From a60593164e82057930d5e9da5173207700413197 Mon Sep 17 00:00:00 2001 From: Roxie Linden Date: Tue, 12 Dec 2023 20:46:34 -0800 Subject: Better renegotiation support for parcel voice Better handle starting up and shutting down WebRTC connections simultaneously. --- indra/llwebrtc/llwebrtc.cpp | 84 ++++++++++-------- indra/llwebrtc/llwebrtc.h | 1 + indra/newview/llvoicewebrtc.cpp | 183 +++++++++++++++++++++++++++------------- indra/newview/llvoicewebrtc.h | 8 +- 4 files changed, 180 insertions(+), 96 deletions(-) diff --git a/indra/llwebrtc/llwebrtc.cpp b/indra/llwebrtc/llwebrtc.cpp index 12997e34d3..e55d94d128 100644 --- a/indra/llwebrtc/llwebrtc.cpp +++ b/indra/llwebrtc/llwebrtc.cpp @@ -83,9 +83,10 @@ void LLAudioDeviceObserver::OnRenderData(const void *audio_samples, void LLWebRTCImpl::init() { RTC_DCHECK(mPeerConnectionFactory); - mPlayoutDevice = -1; - mRecordingDevice = -1; + mPlayoutDevice = 0; + mRecordingDevice = 0; rtc::InitializeSSL(); + rtc::LogMessage::LogToDebug(rtc::LS_WARNING); mTaskQueueFactory = webrtc::CreateDefaultTaskQueueFactory(); mNetworkThread = rtc::Thread::CreateWithSocketServer(); @@ -145,10 +146,9 @@ void LLWebRTCImpl::init() mPeerDeviceModule->InitSpeaker(); mPeerDeviceModule->InitRecording(); mPeerDeviceModule->InitPlayout(); - mPeerDeviceModule->StartPlayout(); - mPeerDeviceModule->StartRecording(); }); - + + apm->ApplyConfig(apm_config); mPeerConnectionFactory = webrtc::CreatePeerConnectionFactory(mNetworkThread.get(), mWorkerThread.get(), mSignalingThread.get(), @@ -159,7 +159,12 @@ void LLWebRTCImpl::init() nullptr /* video_decoder_factory */, nullptr /* audio_mixer */, apm); - apm->ApplyConfig(apm_config); + mWorkerThread->BlockingCall( + [this]() + { + mPeerDeviceModule->StartPlayout(); + mPeerDeviceModule->StartRecording(); + }); } void LLWebRTCImpl::terminate() @@ -235,15 +240,14 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) mTuningDeviceModule->InitMicrophone(); mTuningDeviceModule->InitRecording(); mTuningDeviceModule->StartRecording(); - bool was_peer_recording = false; if (mPeerDeviceModule) { - int16_t captureDeviceCount = mTuningDeviceModule->RecordingDevices(); + int16_t captureDeviceCount = mPeerDeviceModule->RecordingDevices(); for (int16_t i = 0; i < captureDeviceCount; i++) { char name[webrtc::kAdmMaxDeviceNameSize]; char guid[webrtc::kAdmMaxGuidSize]; - mTuningDeviceModule->RecordingDeviceName(i, name, guid); + mPeerDeviceModule->RecordingDeviceName(i, name, guid); if (id == guid || id == "Default") // first one in list is default { RTC_LOG(LS_INFO) @@ -252,11 +256,13 @@ void LLWebRTCImpl::setCaptureDevice(const std::string &id) break; } } - was_peer_recording = mPeerDeviceModule->Recording(); + bool was_peer_recording = mPeerDeviceModule->Recording(); if (was_peer_recording) { mPeerDeviceModule->StopRecording(); } + + mPeerDeviceModule->StopRecording(); mPeerDeviceModule->SetRecordingDevice(mRecordingDevice); mPeerDeviceModule->InitMicrophone(); mPeerDeviceModule->InitRecording(); @@ -432,7 +438,15 @@ void LLWebRTCPeerConnectionImpl::init(LLWebRTCImpl * webrtc_impl) } void LLWebRTCPeerConnectionImpl::terminate() { - shutdownConnection(); + mWebRTCImpl->SignalingBlockingCall( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + }); } void LLWebRTCPeerConnectionImpl::setSignalingObserver(LLWebRTCSignalingObserver *observer) { mSignalingObserverList.emplace_back(observer); } @@ -477,7 +491,6 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() else { RTC_LOG(LS_ERROR) << __FUNCTION__ << "Error creating peer connection: " << error_or_peer_connection.error().message(); - shutdownConnection(); return false; } @@ -546,25 +559,19 @@ bool LLWebRTCPeerConnectionImpl::initializeConnection() void LLWebRTCPeerConnectionImpl::shutdownConnection() { - mWebRTCImpl->SignalingBlockingCall( - [this]() - { - if (mPeerConnection) - { - mPeerConnection->Close(); - mPeerConnection = nullptr; - } - }); - - mWebRTCImpl->NetworkBlockingCall( - [this]() - { - if (mDataChannel) - { - mDataChannel->Close(); - mDataChannel = nullptr; - } - }); + mWebRTCImpl->PostSignalingTask( + [this]() + { + if (mPeerConnection) + { + mPeerConnection->Close(); + mPeerConnection = nullptr; + } + for (auto &observer : mSignalingObserverList) + { + observer->OnPeerShutDown(); + } + }); } void LLWebRTCPeerConnectionImpl::enableSenderTracks(bool enable) @@ -638,15 +645,18 @@ void LLWebRTCPeerConnectionImpl::setReceiveVolume(float volume) mWebRTCImpl->PostSignalingTask( [this, volume]() { - auto receivers = mPeerConnection->GetReceivers(); - - for (auto &receiver : receivers) + if (mPeerConnection) { - for (auto &stream : receiver->streams()) + auto receivers = mPeerConnection->GetReceivers(); + + for (auto &receiver : receivers) { - for (auto &track : stream->GetAudioTracks()) + for (auto &stream : receiver->streams()) { - track->GetSource()->SetVolume(volume); + for (auto &track : stream->GetAudioTracks()) + { + track->GetSource()->SetVolume(volume); + } } } } diff --git a/indra/llwebrtc/llwebrtc.h b/indra/llwebrtc/llwebrtc.h index 9766f2f231..f84e998245 100644 --- a/indra/llwebrtc/llwebrtc.h +++ b/indra/llwebrtc/llwebrtc.h @@ -125,6 +125,7 @@ class LLWebRTCSignalingObserver virtual void OnRenegotiationNeeded() = 0; virtual void OnAudioEstablished(LLWebRTCAudioInterface *audio_interface) = 0; virtual void OnDataChannelReady(LLWebRTCDataInterface *data_interface) = 0; + virtual void OnPeerShutDown() = 0; }; class LLWebRTCPeerConnection diff --git a/indra/newview/llvoicewebrtc.cpp b/indra/newview/llvoicewebrtc.cpp index 9f290b9d4a..ec1e6a353d 100644 --- a/indra/newview/llvoicewebrtc.cpp +++ b/indra/newview/llvoicewebrtc.cpp @@ -86,10 +86,10 @@ extern void handle_voice_morphing_subscribe(); namespace { const F32 VOLUME_SCALE_WEBRTC = 0.01f; - const F32 LEVEL_SCALE_WEBRTC = 0.01f; + const F32 LEVEL_SCALE_WEBRTC = 0.008f; const F32 SPEAKING_TIMEOUT = 1.f; - const F32 SPEAKING_AUDIO_LEVEL = 0.10; + const F32 SPEAKING_AUDIO_LEVEL = 0.40; static const std::string VOICE_SERVER_TYPE = "WebRTC"; @@ -121,9 +121,19 @@ namespace { // Maximum length of capture buffer recordings in seconds. const F32 CAPTURE_BUFFER_MAX_TIME = 10.f; +} // namespace +float LLWebRTCVoiceClient::getAudioLevel() +{ + if (mIsInTuningMode) + { + return (1.0 - mWebRTCDeviceInterface->getTuningAudioLevel() * LEVEL_SCALE_WEBRTC) * mTuningMicGain / 2.1; + } + else + { + return (1.0 - mWebRTCDeviceInterface->getPeerAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1; + } } - /////////////////////////////////////////////////////////////////////////////////////////////// void LLVoiceWebRTCStats::reset() @@ -350,7 +360,7 @@ void LLWebRTCVoiceClient::cleanUp() mNextAudioSession.reset(); mAudioSession.reset(); - sessionState::deleteAllSessions(); + sessionState::for_each(boost::bind(predShutdownSession, _1)); LL_DEBUGS("Voice") << "exiting" << LL_ENDL; } @@ -393,6 +403,14 @@ void LLWebRTCVoiceClient::predOnConnectionFailure(const LLWebRTCVoiceClient::ses void LLWebRTCVoiceClient::OnConnectionFailure(const std::string& channelID) { + if (mNextAudioSession && mNextAudioSession->mChannelID == channelID) + { + LLWebRTCVoiceClient::getInstance()->notifyStatusObservers(LLVoiceClientStatusObserver::ERROR_UNKNOWN); + } + else if (mAudioSession && mAudioSession->mChannelID == channelID) + { + LLWebRTCVoiceClient::getInstance()->notifyStatusObservers(LLVoiceClientStatusObserver::ERROR_UNKNOWN); + } sessionState::for_each(boost::bind(predOnConnectionFailure, _1, channelID)); } @@ -468,10 +486,6 @@ void LLWebRTCVoiceClient::voiceConnectionCoro() while (!sShuttingDown) { llcoro::suspendUntilTimeout(UPDATE_THROTTLE_SECONDS); - if (!mVoiceEnabled) - { - continue; - } // add session for region or parcel voice. LLViewerRegion *regionp = gAgent.getRegion(); if (!regionp) @@ -482,7 +496,7 @@ void LLWebRTCVoiceClient::voiceConnectionCoro() { continue; } - if (!mAudioSession || mAudioSession->mIsSpatial) + if (mVoiceEnabled && (!mAudioSession || mAudioSession->mIsSpatial)) { // check to see if parcel changed. std::string channelID = regionp->getRegionID().asString(); @@ -499,11 +513,14 @@ void LLWebRTCVoiceClient::voiceConnectionCoro() setSpatialChannel(channelID, "", parcel_local_id); } } - updatePosition(); sessionState::for_each(boost::bind(predProcessSessionStates, _1)); - sessionState::reapEmptySessions(); - sendPositionAndVolumeUpdate(true); - updateOwnVolume(); + reapEmptySessions(); + if (mVoiceEnabled) + { + updatePosition(); + sendPositionAndVolumeUpdate(true); + updateOwnVolume(); + } } } catch (const LLCoros::Stop&) @@ -526,15 +543,6 @@ void LLWebRTCVoiceClient::voiceConnectionCoro() cleanUp(); } -bool LLWebRTCVoiceClient::performMicTuning() -{ - LL_INFOS("Voice") << "Entering voice tuning mode." << LL_ENDL; - - mIsInTuningMode = false; - - //--------------------------------------------------------------------- - return true; -} //========================================================================= @@ -665,10 +673,11 @@ void LLWebRTCVoiceClient::tuningSetSpeakerVolume(float volume) mTuningSpeakerVolumeDirty = true; } } - + + float LLWebRTCVoiceClient::tuningGetEnergy(void) { - return (1.0 - mWebRTCDeviceInterface->getTuningAudioLevel() * LEVEL_SCALE_WEBRTC) * mTuningMicGain/2.1; + return getAudioLevel(); } bool LLWebRTCVoiceClient::deviceSettingsAvailable() @@ -730,7 +739,7 @@ void LLWebRTCVoiceClient::sendPositionAndVolumeUpdate(bool force) if (!mMuteMic && !mTuningMode) { - audio_level = (1.0 - mWebRTCDeviceInterface->getTuningAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1; + audio_level = getAudioLevel(); uint_audio_level = (uint32_t) (audio_level*128); } @@ -801,8 +810,8 @@ void LLWebRTCVoiceClient::updateOwnVolume() { F32 audio_level = 0.0; if (!mMuteMic && !mTuningMode) { - audio_level = (F32) (1.0 - mWebRTCDeviceInterface->getTuningAudioLevel() * LEVEL_SCALE_WEBRTC) * mMicGain / 2.1; - LL_DEBUGS("Voice") << "Level " << audio_level << LL_ENDL; + audio_level = getAudioLevel(); + LL_WARNS("Voice") << "Level " << audio_level << LL_ENDL; } sessionState::for_each(boost::bind(predUpdateOwnVolume, _1, audio_level)); @@ -2052,6 +2061,22 @@ BOOL LLWebRTCVoiceClient::getAreaVoiceDisabled() return mAreaVoiceDisabled; } +void LLWebRTCVoiceClient::reapEmptySessions() +{ + sessionState::reapEmptySessions(); + + // mAudioSession or mNextAudioSession was reaped, + // so reset them. + if (mAudioSession && !sessionState::hasSession(mAudioSession->mChannelID)) + { + mAudioSession.reset(); + } + if (mNextAudioSession && !sessionState::hasSession(mNextAudioSession->mChannelID)) + { + mNextAudioSession.reset(); + } +} + //------------------------------------------------------------------------ std::map LLWebRTCVoiceClient::sessionState::mSessions; @@ -2241,6 +2266,11 @@ LLWebRTCVoiceClient::sessionStatePtr_t LLWebRTCVoiceClient::addSession(const std return result; } +void LLWebRTCVoiceClient::predShutdownSession(const LLWebRTCVoiceClient::sessionStatePtr_t& session) +{ + session->shutdownAllConnections(); +} + void LLWebRTCVoiceClient::deleteSession(const sessionStatePtr_t &session) { // At this point, the session should be unhooked from all lists and all state should be consistent. @@ -2260,11 +2290,6 @@ void LLWebRTCVoiceClient::deleteSession(const sessionStatePtr_t &session) } } -void LLWebRTCVoiceClient::sessionState::deleteAllSessions() -{ - mSessions.clear(); -} - void LLWebRTCVoiceClient::verifySessionState(void) { sessionState::VerifySessions(); @@ -2356,6 +2381,8 @@ void LLWebRTCVoiceClient::notifyStatusObservers(LLVoiceClientStatusObserver::ESt << ", proximal is " << inSpatialChannel() << LL_ENDL; + mIsProcessingChannels = status == LLVoiceClientStatusObserver::STATUS_LOGGED_IN; + for (status_observer_set_t::iterator it = mStatusObservers.begin(); it != mStatusObservers.end(); ) @@ -2365,10 +2392,7 @@ void LLWebRTCVoiceClient::notifyStatusObservers(LLVoiceClientStatusObserver::ESt // In case onError() deleted an entry. it = mStatusObservers.upper_bound(observer); } - mIsProcessingChannels = status == LLVoiceClientStatusObserver::STATUS_LOGGED_IN; - { - } // skipped to avoid speak button blinking if ( status != LLVoiceClientStatusObserver::STATUS_JOINING && status != LLVoiceClientStatusObserver::STATUS_LEFT_CHANNEL @@ -2483,6 +2507,7 @@ LLVoiceWebRTCConnection::~LLVoiceWebRTCConnection() // by llwebrtc::terminate() on shutdown. return; } + assert(mOutstandingRequests == 0); mWebRTCPeerConnection->unsetSignalingObserver(this); llwebrtc::freePeerConnection(mWebRTCPeerConnection); mWebRTCPeerConnection = nullptr; @@ -2544,14 +2569,16 @@ void LLVoiceWebRTCConnection::onIceUpdateError(int retries, std::string url, LLS body, boost::bind(&LLVoiceWebRTCConnection::onIceUpdateComplete, this, ice_completed, _1), boost::bind(&LLVoiceWebRTCConnection::onIceUpdateError, this, retries - 1, url, body, ice_completed, _1)); - mOutstandingRequests++; + return; } - else + + LL_WARNS("Voice") << "Unable to complete ice trickling voice account, restarting connection. " << result << LL_ENDL; + if (!mShutDown) { - LL_WARNS("Voice") << "Unable to complete ice trickling voice account, restarting connection. " << result << LL_ENDL; setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); - mTrickling = false; } + mTrickling = false; + mOutstandingRequests--; } @@ -2655,7 +2682,16 @@ void LLVoiceWebRTCConnection::OnDataChannelReady(llwebrtc::LLWebRTCDataInterface void LLVoiceWebRTCConnection::OnRenegotiationNeeded() { LL_INFOS("Voice") << "On Renegotiation Needed." << LL_ENDL; - setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + if (!mShutDown) + { + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + } +} + +void LLVoiceWebRTCConnection::OnPeerShutDown() +{ + setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); + mOutstandingRequests--; } void LLVoiceWebRTCConnection::processIceUpdates() @@ -2664,6 +2700,10 @@ void LLVoiceWebRTCConnection::processIceUpdates() { return; } + if (mShutDown) + { + return; + } LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID); if (!regionp || !regionp->capabilitiesReceived()) { @@ -2814,26 +2854,25 @@ void LLVoiceWebRTCConnection::OnVoiceConnectionRequestFailure(std::string url, i body, boost::bind(&LLVoiceWebRTCConnection::OnVoiceConnectionRequestSuccess, this, _1), boost::bind(&LLVoiceWebRTCConnection::OnVoiceConnectionRequestFailure, this, url, retries - 1, body, _1)); - mOutstandingRequests++; - } - else - { - LL_WARNS("Voice") << "Unable to connect voice." << result << LL_ENDL; - setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); + return; } + LL_WARNS("Voice") << "Unable to connect voice." << result << LL_ENDL; + setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); mOutstandingRequests--; } bool LLVoiceWebRTCConnection::connectionStateMachine() -{ - U32 retry = 0; - +{ processIceUpdates(); switch (getVoiceConnectionState()) { case VOICE_STATE_START_SESSION: { + if (mShutDown) + { + setVoiceConnectionState(VOICE_STATE_DISCONNECT); + } mTrickling = false; mIceCompleted = false; setVoiceConnectionState(VOICE_STATE_WAIT_FOR_SESSION_START); @@ -2845,9 +2884,18 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() } case VOICE_STATE_WAIT_FOR_SESSION_START: { + if (mShutDown) + { + setVoiceConnectionState(VOICE_STATE_DISCONNECT); + } break; } case VOICE_STATE_REQUEST_CONNECTION: + if (mShutDown) + { + setVoiceConnectionState(VOICE_STATE_DISCONNECT); + break; + } if (!requestVoiceConnection()) { setVoiceConnectionState(VOICE_STATE_SESSION_RETRY); @@ -2858,10 +2906,19 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() } break; case VOICE_STATE_CONNECTION_WAIT: + if (mShutDown) + { + setVoiceConnectionState(VOICE_STATE_DISCONNECT); + } break; case VOICE_STATE_SESSION_ESTABLISHED: { + if (mShutDown) + { + setVoiceConnectionState(VOICE_STATE_DISCONNECT); + break; + } mWebRTCAudioInterface->setMute(mMuted); mWebRTCAudioInterface->setReceiveVolume(mSpeakerVolume); mWebRTCAudioInterface->setSendVolume(mMicGain); @@ -2886,8 +2943,6 @@ bool LLVoiceWebRTCConnection::connectionStateMachine() case VOICE_STATE_DISCONNECT: breakVoiceConnection(true); - - retry = 0; // Connected without issues break; case VOICE_STATE_WAIT_FOR_EXIT: @@ -2934,10 +2989,6 @@ bool LLVoiceWebRTCConnection::breakVoiceConnection(bool corowait) mWebRTCDataInterface = nullptr; } mWebRTCAudioInterface = nullptr; - if (mWebRTCPeerConnection) - { - mWebRTCPeerConnection->shutdownConnection(); - } LLViewerRegion *regionp = LLWorld::instance().getRegionFromID(mRegionID); if (!regionp || !regionp->capabilitiesReceived()) { @@ -2967,6 +3018,7 @@ bool LLVoiceWebRTCConnection::breakVoiceConnection(bool corowait) body, boost::bind(&LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestSuccess, this, _1), boost::bind(&LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestFailure, this, url, 3, body, _1)); + setVoiceConnectionState(VOICE_STATE_WAIT_FOR_EXIT); mOutstandingRequests++; return true; } @@ -2977,7 +3029,16 @@ void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestSuccess(const LLSD &res { return; } - setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); + + if (mWebRTCPeerConnection) + { + mOutstandingRequests++; + mWebRTCPeerConnection->shutdownConnection(); + } + else + { + setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); + } mOutstandingRequests--; } @@ -2995,9 +3056,17 @@ void LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestFailure(std::string url body, boost::bind(&LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestSuccess, this, _1), boost::bind(&LLVoiceWebRTCConnection::OnVoiceDisconnectionRequestFailure, this, url, retries - 1, body, _1)); + return; + } + if (mWebRTCPeerConnection) + { mOutstandingRequests++; + mWebRTCPeerConnection->shutdownConnection(); + } + else + { + setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); } - setVoiceConnectionState(VOICE_STATE_SESSION_EXIT); mOutstandingRequests--; } diff --git a/indra/newview/llvoicewebrtc.h b/indra/newview/llvoicewebrtc.h index 46faf52da5..f311f241dc 100644 --- a/indra/newview/llvoicewebrtc.h +++ b/indra/newview/llvoicewebrtc.h @@ -381,7 +381,7 @@ public: LLUUID mVoiceFontID; static void VerifySessions(); - static void deleteAllSessions(); + static bool hasSession(const std::string &sessionID) { return mSessions.find(sessionID) != mSessions.end(); } private: @@ -414,6 +414,8 @@ public: static void predSetMuteMic(const LLWebRTCVoiceClient::sessionStatePtr_t &session, bool mute); static void predSetMicGain(const LLWebRTCVoiceClient::sessionStatePtr_t &session, F32 volume); static void predSetSpeakerVolume(const LLWebRTCVoiceClient::sessionStatePtr_t &session, F32 volume); + static void predShutdownSession(const LLWebRTCVoiceClient::sessionStatePtr_t &session); + void reapEmptySessions(); ////////////////////////////// /// @name TVC/Server management and communication @@ -535,6 +537,8 @@ public: void accountGetTemplateFontsResponse(int statusCode, const std::string &statusString); private: + + float getAudioLevel(); LLVoiceVersionInfo mVoiceVersion; @@ -544,7 +548,6 @@ private: void voiceConnectionStateMachine(); - bool performMicTuning(); //--- /// Clean up objects created during a voice session. void cleanUp(); @@ -732,6 +735,7 @@ class LLVoiceWebRTCConnection : void OnOfferAvailable(const std::string &sdp) override; void OnRenegotiationNeeded() override; void OnAudioEstablished(llwebrtc::LLWebRTCAudioInterface *audio_interface) override; + void OnPeerShutDown() override; //@} ///////////////////////// -- cgit v1.2.3