feat: Renegotiation

NextcloudTalk/Calls/CallKitManager.swift

@@ -312,6 +312,12 @@ public class CallKitManager: NSObject, CXProviderDelegate {
         self.provider.reportCall(with: uuid, updated: update)
     }
 
+    public func changeHasVideo(_ hasVideo: Bool, forCall token: String) {
+        guard let call = self.call(forToken: token) else { return }
+
+        self.updateCall(call, hasVideo: hasVideo)
+    }
+
     private func stopHangUpTimer(forCallUUID uuid: UUID) {
         if let hangUpTimer = self.hangUpTimers[uuid] {
             hangUpTimer.invalidate()

NextcloudTalk/Calls/CallViewController.swift

@@ -1689,6 +1689,13 @@ class CallViewController: UIViewController,
             items.append(reactionMenu)
         }
 
+        // Switch back to a voice only call
+        if !self.isAudioOnly, callController.supportsCallUpgradeUsingRenegotiation {
+            items.append(UIAction(title: NSLocalizedString("Switch to voice-only call", comment: ""), image: .init(systemName: "video.slash"), handler: { [unowned self] _ in
+                self.switchToVoiceOnlyCall()
+            }))
+        }
+
         // Background blur
         if !self.isAudioOnly, self.room.canPublishVideo {
             var blurActionImage = UIImage(systemName: "person.and.background.dotted")
@@ -2101,7 +2108,12 @@ class CallViewController: UIViewController,
 
     func showUpgradeToVideoCallDialog() {
         let confirmTitle = NSLocalizedString("Do you want to enable your camera?", comment: "")
-        let confirmMessage = NSLocalizedString("If you enable your camera, this call will be interrupted for a few seconds.", comment: "")
+        var confirmMessage: String?
+
+        // Without renegotiation support the call needs to be restarted to be upgraded to a video call
+        if callController?.supportsCallUpgradeUsingRenegotiation != true {
+            confirmMessage = NSLocalizedString("If you enable your camera, this call will be interrupted for a few seconds.", comment: "")
+        }
 
         let confirmDialog = UIAlertController(title: confirmTitle, message: confirmMessage, preferredStyle: .alert)
         confirmDialog.addAction(UIAlertAction(title: NSLocalizedString("Enable", comment: ""), style: .default) { _ in
@@ -2113,8 +2125,63 @@ class CallViewController: UIViewController,
     }
 
     func upgradeToVideoCall() {
-        self.videoCallUpgrade = true
-        self.hangup(forAll: false)
+        if callController?.supportsCallUpgradeUsingRenegotiation == true {
+            self.upgradeToVideoCallUsingRenegotiation()
+        } else {
+            // Without renegotiation support the call needs to be restarted with video enabled
+            self.videoCallUpgrade = true
+            self.hangup(forAll: false)
+        }
+    }
+
+    func upgradeToVideoCallUsingRenegotiation() {
+        guard let callController else { return }
+
+        let authStatus = AVCaptureDevice.authorizationStatus(for: .video)
+
+        if authStatus == .notDetermined {
+            AVCaptureDevice.requestAccess(for: .video) { _ in
+                DispatchQueue.main.async {
+                    self.upgradeToVideoCallUsingRenegotiation()
+                }
+            }
+
+            return
+        }
+
+        self.isAudioOnly = false
+
+        // In voice only calls the video disable button was disabled when no local video track was
+        // created (see didCreateLocalVideoTrack delegate), so it needs to be enabled again
+        if callController.isCameraAccessAvailable(), room.canPublishVideo {
+            self.userDisabledVideo = false
+            self.setVideoDisableButtonEnabled(true)
+        }
+
+        callController.upgradeToVideoCall()
+        self.adjustBars()
+    }
+
+    func switchToVoiceOnlyCall() {
+        guard let callController else { return }
+
+        self.isAudioOnly = true
+        self.userDisabledVideo = true
+
+        callController.downgradeToVoiceOnlyCall()
+
+        self.setLocalVideoViewWrapperHidden(true)
+
+        // Remote videos are no longer received, so show the avatars of all participants again
+        DispatchQueue.main.async {
+            for peer in self.peersInCall where peer.roomType == kRoomTypeVideo {
+                self.updatePeer(peer) { cell in
+                    cell.videoDisabled = true
+                }
+            }
+        }
+
+        self.adjustBars()
     }
 
     @IBAction func toggleChatButtonPressed(_ sender: Any) {

NextcloudTalk/Calls/NCCallController.swift

@@ -48,7 +48,7 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling
     private let room: NCRoom
     private let account: TalkAccount
     private let userSessionId: String
-    private let isAudioOnly: Bool
+    private var isAudioOnly: Bool
 
     // TODO: Default true?
     public var disableAudioAtStart: Bool = false
@@ -454,6 +454,126 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling
         self.cameraController?.switchCamera()
     }
 
+    // Upgrading a voice only call using renegotiation is only supported when using a MCU
+    // and the signaling server supports updating existing connections ("update-sdp").
+    // Otherwise the call needs to be restarted to be upgraded to a video call.
+    public var supportsCallUpgradeUsingRenegotiation: Bool {
+        guard let externalSignalingController else { return false }
+
+        return externalSignalingController.hasMCU && externalSignalingController.hasUpdateSdp
+    }
+
+    public func upgradeToVideoCall() {
+        WebRTCCommon.shared.dispatch {
+            guard self.isAudioOnly, self.supportsCallUpgradeUsingRenegotiation else { return }
+
+            NCLog.log("Upgrading voice only call to video call using renegotiation")
+
+            self.isAudioOnly = false
+            self.disableVideoAtStart = false
+
+            NCAudioController.shared.setAudioSessionToVideoChatMode()
+
+            // Create the local video track, since it is not created in voice only calls
+            if self.localVideoTrack == nil, self.room.canPublishVideo, self.isCameraAccessAvailable() {
+                self.createLocalVideoTrack()
+            }
+
+            if let localVideoTrack = self.localVideoTrack {
+                if let peerConnection = self.publisherPeerConnection?.getPeerConnection() {
+                    if let videoTransceiver = peerConnection.transceivers.first(where: { $0.mediaType == .video }) {
+                        // The video m-line was already negotiated with a placeholder transceiver when
+                        // the publisher peer connection was created, so the video track can start to
+                        // be sent right away, without any renegotiation
+                        videoTransceiver.sender.track = localVideoTrack
+                    } else {
+                        // The publisher peer connection was negotiated without a video m-line. Adding
+                        // one to an existing connection is not supported by the MCU (see comment in
+                        // createPublisherPeerConnection), so the call needs to be reconnected instead
+                        self.forceReconnect()
+                        return
+                    }
+                } else {
+                    // There was no publisher peer connection (e.g. no microphone access), create it now
+                    self.createPublisherPeerConnection()
+                }
+            }
+
+            // Update the existing receiver peer connections, so remote videos can be received.
+            // Requesting an offer with the "sid" of the existing connection makes the MCU update
+            // that connection with a renegotiation offer instead of creating a new one.
+            for (_, peerConnectionWrapper) in self.connectionsDict {
+                guard !peerConnectionWrapper.isMCUPublisherPeer, !peerConnectionWrapper.isDummyPeer,
+                      peerConnectionWrapper.roomType == kRoomTypeVideo
+                else { continue }
+
+                peerConnectionWrapper.isAudioOnly = false
+                self.requestOfferWithRepetition(forSessionId: peerConnectionWrapper.peerId, withRoomType: kRoomTypeVideo, withSid: peerConnectionWrapper.sid)
+            }
+
+            // Let the other participants know that we are now in the call with video
+            self.updateCallFlagsInServer()
+            self.startSendingCurrentState()
+
+            if CallKitManager.isCallKitAvailable() {
+                CallKitManager.sharedInstance().changeHasVideo(true, forCall: self.room.token)
+            }
+        }
+    }
+
+    public func downgradeToVoiceOnlyCall() {
+        WebRTCCommon.shared.dispatch {
+            guard !self.isAudioOnly, self.supportsCallUpgradeUsingRenegotiation else { return }
+
+            NCLog.log("Downgrading video call to voice only call using renegotiation")
+
+            self.isAudioOnly = true
+            self.disableVideoAtStart = true
+
+            NCAudioController.shared.setAudioSessionToVoiceChatMode()
+
+            // Stop sending the local video, but keep the negotiated m-line (a sender without
+            // a track), so the call can be upgraded to a video call again later
+            if let peerConnection = self.publisherPeerConnection?.getPeerConnection(),
+               let videoTransceiver = peerConnection.transceivers.first(where: { $0.mediaType == .video }) {
+                videoTransceiver.sender.track = nil
+            }
+
+            // Stop capturing and release the local video track and the camera
+            self.cameraController?.stopAVCaptureSession()
+            self.cameraController = nil
+            self.localVideoTrack = nil
+
+            // Stop receiving remote videos. Requesting an offer with the "sid" of the existing
+            // connection makes the MCU update that connection with a renegotiation offer, which
+            // is then answered with inactive video transceivers, since the call is audio only again.
+            for (_, peerConnectionWrapper) in self.connectionsDict {
+                guard !peerConnectionWrapper.isMCUPublisherPeer, !peerConnectionWrapper.isDummyPeer,
+                      peerConnectionWrapper.roomType == kRoomTypeVideo
+                else { continue }
+
+                peerConnectionWrapper.isAudioOnly = true
+                self.requestOfferWithRepetition(forSessionId: peerConnectionWrapper.peerId, withRoomType: kRoomTypeVideo, withSid: peerConnectionWrapper.sid)
+            }
+
+            // Let the other participants know that we are now in the call without video
+            self.updateCallFlagsInServer()
+            self.startSendingCurrentState()
+
+            if CallKitManager.isCallKitAvailable() {
+                CallKitManager.sharedInstance().changeHasVideo(false, forCall: self.room.token)
+            }
+        }
+    }
+
+    private func updateCallFlagsInServer() {
+        NCAPIController.shared.updateCallFlags(inRoom: self.room.token, withCallFlags: self.joinCallFlags, forAccount: self.account) { error in
+            if let error {
+                NCLog.log("Could not update call flags. Error: \(error)")
+            }
+        }
+    }
+
     public func enableVideo(_ enable: Bool) {
         WebRTCCommon.shared.dispatch {
             if enable {
@@ -1025,6 +1145,15 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling
 
         if let localVideoTrack {
             peerConnection.add(localVideoTrack, streamIds: [NCCallController.kNCMediaStreamId])
+        } else if self.room.canPublishVideo {
+            // In voice only calls, already negotiate a placeholder video m-line (a transceiver
+            // without a track), so the call can later be upgraded to a video call by just replacing
+            // the sender track. A renegotiation that adds a new m-line to an existing publisher
+            // peer connection completes on the SDP level, but the MCU never relays its media.
+            let transceiverInit = RTCRtpTransceiverInit()
+            transceiverInit.direction = .sendOnly
+            transceiverInit.streamIds = [NCCallController.kNCMediaStreamId]
+            peerConnection.addTransceiver(of: .video, init: transceiverInit)
         }
 
         peerConnectionWrapper.sendPublisherOffer()
@@ -1060,24 +1189,49 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling
         peerConnectionWrapper.sendPublisherOffer()
     }
 
-    public func requestOfferWithRepetition(forSessionId sessionId: String, withRoomType roomType: String) {
+    public func requestOfferWithRepetition(forSessionId sessionId: String, withRoomType roomType: String, withSid sid: String? = nil) {
         WebRTCCommon.shared.assertQueue()
 
+        // Requesting an offer with a "sid" updates the existing connection. If that connection has
+        // failed it would require an ICE restart rather than an update to recover, so request a new
+        // connection instead (by omitting the "sid"). The offer will then arrive with a new "sid",
+        // which replaces the failed peer connection (see processOfferAnswer).
+        let isConnectionFailed = self.getPeerConnectionWrapper(forSessionId: sessionId, ofType: roomType)?.isConnectionFailed() ?? false
+        let updateSid = isConnectionFailed ? nil : sid
+
+        let peerKey = self.getPeerKey(withSessionId: sessionId, ofType: roomType, forOwnScreenshare: false)
+
+        if let previousOfferTimer = self.pendingOffersDict[peerKey], previousOfferTimer.isValid {
+            // If a connection needs to be updated ("sid" is set) but another offer request is already
+            // pending, ignore the new update. A new connection needs to be requested even if there
+            // is another one already pending.
+            if updateSid != nil {
+                return
+            }
+
+            DispatchQueue.main.async {
+                previousOfferTimer.invalidate()
+            }
+        }
+
         let timeout = Int(Date().timeIntervalSince1970 + 60)
-        let userInfo: [String: Any] = [
+        var userInfo: [String: Any] = [
             "sessionId": sessionId,
             "roomType": roomType,
             "timeout": timeout
         ]
 
+        if let updateSid {
+            userInfo["sid"] = updateSid
+        }
+
         let pendingOfferTimer = Timer(timeInterval: 8.0, target: self, selector: #selector(self.requestNewOffer), userInfo: userInfo, repeats: true)
-        let peerKey = self.getPeerKey(withSessionId: sessionId, ofType: roomType, forOwnScreenshare: false)
 
         self.pendingOffersDict[peerKey] = pendingOfferTimer
 
         // Request new offer
         if let externalSignalingController {
-            externalSignalingController.requestOffer(forSessionId: sessionId, andRoomType: roomType)
+            externalSignalingController.requestOffer(forSessionId: sessionId, andRoomType: roomType, withSid: updateSid)
         }
 
         DispatchQueue.main.async {
@@ -1093,10 +1247,12 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling
               let externalSignalingController
         else { return }
 
+        let sid = userInfo["sid"] as? String
+
         WebRTCCommon.shared.dispatch {
             if Int(Date().timeIntervalSince1970) < timeout {
                 print("Re-requesting an offer to session \(sessionId)")
-                externalSignalingController.requestOffer(forSessionId: sessionId, andRoomType: roomType)
+                externalSignalingController.requestOffer(forSessionId: sessionId, andRoomType: roomType, withSid: sid)
             } else {
                 DispatchQueue.main.async {
                     timer.invalidate()
@@ -1323,6 +1479,8 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling
     }
 
     func peerConnection(_ peerConnection: NCPeerConnection, didRemove stream: RTCMediaStream?) {
+        // The stream is nil when a receiver was removed (e.g. when a video transceiver
+        // was set to inactive in an audio only call)
         guard !peerConnection.isMCUPublisherPeer, let stream else { return }
 
         self.delegate?.callController(self, didRemoveStream: stream, ofPeer: peerConnection)

NextcloudTalk/Network/NCAPIController.swift

@@ -1924,6 +1924,19 @@ class NCAPIController: NSObject, NKCommonDelegate {
         }
     }
 
+    @discardableResult
+    public func updateCallFlags(inRoom token: String, withCallFlags flags: CallFlag, forAccount account: TalkAccount, completionBlock: @escaping (_ error: OcsError?) -> Void) -> URLSessionDataTask? {
+        guard let apiSessionManager = self.getAPISessionManager(forAccountId: account.accountId),
+              let encodedToken = token.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed)
+        else { return nil }
+
+        let urlString = self.getRequestURL(forEndpoint: "call/\(encodedToken)", withAPIType: .call, forAccount: account)
+
+        return apiSessionManager.putOcs(urlString, account: account, parameters: ["flags": flags.rawValue]) { _, ocsError in
+            completionBlock(ocsError)
+        }
+    }
+
     @discardableResult
     public func leaveCall(inRoom token: String, forAllParticipants allParticipants: Bool, forAccount account: TalkAccount, completionBlock: @escaping (_ error: OcsError?) -> Void) -> URLSessionDataTask? {
         guard let apiSessionManager = self.getAPISessionManager(forAccountId: account.accountId),