diff --git a/NextcloudTalk/Calls/CallKitManager.swift b/NextcloudTalk/Calls/CallKitManager.swift index e82dffdbd..8f6b33685 100644 --- a/NextcloudTalk/Calls/CallKitManager.swift +++ b/NextcloudTalk/Calls/CallKitManager.swift @@ -312,6 +312,12 @@ public class CallKitManager: NSObject, CXProviderDelegate { self.provider.reportCall(with: uuid, updated: update) } + public func changeHasVideo(_ hasVideo: Bool, forCall token: String) { + guard let call = self.call(forToken: token) else { return } + + self.updateCall(call, hasVideo: hasVideo) + } + private func stopHangUpTimer(forCallUUID uuid: UUID) { if let hangUpTimer = self.hangUpTimers[uuid] { hangUpTimer.invalidate() diff --git a/NextcloudTalk/Calls/CallViewController.swift b/NextcloudTalk/Calls/CallViewController.swift index 55db91d70..0fd33efae 100644 --- a/NextcloudTalk/Calls/CallViewController.swift +++ b/NextcloudTalk/Calls/CallViewController.swift @@ -1689,6 +1689,13 @@ class CallViewController: UIViewController, items.append(reactionMenu) } + // Switch back to a voice only call + if !self.isAudioOnly, callController.supportsCallUpgradeUsingRenegotiation { + items.append(UIAction(title: NSLocalizedString("Switch to voice-only call", comment: ""), image: .init(systemName: "video.slash"), handler: { [unowned self] _ in + self.switchToVoiceOnlyCall() + })) + } + // Background blur if !self.isAudioOnly, self.room.canPublishVideo { var blurActionImage = UIImage(systemName: "person.and.background.dotted") @@ -2101,7 +2108,12 @@ class CallViewController: UIViewController, func showUpgradeToVideoCallDialog() { let confirmTitle = NSLocalizedString("Do you want to enable your camera?", comment: "") - let confirmMessage = NSLocalizedString("If you enable your camera, this call will be interrupted for a few seconds.", comment: "") + var confirmMessage: String? + + // Without renegotiation support the call needs to be restarted to be upgraded to a video call + if callController?.supportsCallUpgradeUsingRenegotiation != true { + confirmMessage = NSLocalizedString("If you enable your camera, this call will be interrupted for a few seconds.", comment: "") + } let confirmDialog = UIAlertController(title: confirmTitle, message: confirmMessage, preferredStyle: .alert) confirmDialog.addAction(UIAlertAction(title: NSLocalizedString("Enable", comment: ""), style: .default) { _ in @@ -2113,8 +2125,63 @@ class CallViewController: UIViewController, } func upgradeToVideoCall() { - self.videoCallUpgrade = true - self.hangup(forAll: false) + if callController?.supportsCallUpgradeUsingRenegotiation == true { + self.upgradeToVideoCallUsingRenegotiation() + } else { + // Without renegotiation support the call needs to be restarted with video enabled + self.videoCallUpgrade = true + self.hangup(forAll: false) + } + } + + func upgradeToVideoCallUsingRenegotiation() { + guard let callController else { return } + + let authStatus = AVCaptureDevice.authorizationStatus(for: .video) + + if authStatus == .notDetermined { + AVCaptureDevice.requestAccess(for: .video) { _ in + DispatchQueue.main.async { + self.upgradeToVideoCallUsingRenegotiation() + } + } + + return + } + + self.isAudioOnly = false + + // In voice only calls the video disable button was disabled when no local video track was + // created (see didCreateLocalVideoTrack delegate), so it needs to be enabled again + if callController.isCameraAccessAvailable(), room.canPublishVideo { + self.userDisabledVideo = false + self.setVideoDisableButtonEnabled(true) + } + + callController.upgradeToVideoCall() + self.adjustBars() + } + + func switchToVoiceOnlyCall() { + guard let callController else { return } + + self.isAudioOnly = true + self.userDisabledVideo = true + + callController.downgradeToVoiceOnlyCall() + + self.setLocalVideoViewWrapperHidden(true) + + // Remote videos are no longer received, so show the avatars of all participants again + DispatchQueue.main.async { + for peer in self.peersInCall where peer.roomType == kRoomTypeVideo { + self.updatePeer(peer) { cell in + cell.videoDisabled = true + } + } + } + + self.adjustBars() } @IBAction func toggleChatButtonPressed(_ sender: Any) { diff --git a/NextcloudTalk/Calls/NCCallController.swift b/NextcloudTalk/Calls/NCCallController.swift index 2c405ad8c..35d7f61e3 100644 --- a/NextcloudTalk/Calls/NCCallController.swift +++ b/NextcloudTalk/Calls/NCCallController.swift @@ -48,7 +48,7 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling private let room: NCRoom private let account: TalkAccount private let userSessionId: String - private let isAudioOnly: Bool + private var isAudioOnly: Bool // TODO: Default true? public var disableAudioAtStart: Bool = false @@ -454,6 +454,126 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling self.cameraController?.switchCamera() } + // Upgrading a voice only call using renegotiation is only supported when using a MCU + // and the signaling server supports updating existing connections ("update-sdp"). + // Otherwise the call needs to be restarted to be upgraded to a video call. + public var supportsCallUpgradeUsingRenegotiation: Bool { + guard let externalSignalingController else { return false } + + return externalSignalingController.hasMCU && externalSignalingController.hasUpdateSdp + } + + public func upgradeToVideoCall() { + WebRTCCommon.shared.dispatch { + guard self.isAudioOnly, self.supportsCallUpgradeUsingRenegotiation else { return } + + NCLog.log("Upgrading voice only call to video call using renegotiation") + + self.isAudioOnly = false + self.disableVideoAtStart = false + + NCAudioController.shared.setAudioSessionToVideoChatMode() + + // Create the local video track, since it is not created in voice only calls + if self.localVideoTrack == nil, self.room.canPublishVideo, self.isCameraAccessAvailable() { + self.createLocalVideoTrack() + } + + if let localVideoTrack = self.localVideoTrack { + if let peerConnection = self.publisherPeerConnection?.getPeerConnection() { + if let videoTransceiver = peerConnection.transceivers.first(where: { $0.mediaType == .video }) { + // The video m-line was already negotiated with a placeholder transceiver when + // the publisher peer connection was created, so the video track can start to + // be sent right away, without any renegotiation + videoTransceiver.sender.track = localVideoTrack + } else { + // The publisher peer connection was negotiated without a video m-line. Adding + // one to an existing connection is not supported by the MCU (see comment in + // createPublisherPeerConnection), so the call needs to be reconnected instead + self.forceReconnect() + return + } + } else { + // There was no publisher peer connection (e.g. no microphone access), create it now + self.createPublisherPeerConnection() + } + } + + // Update the existing receiver peer connections, so remote videos can be received. + // Requesting an offer with the "sid" of the existing connection makes the MCU update + // that connection with a renegotiation offer instead of creating a new one. + for (_, peerConnectionWrapper) in self.connectionsDict { + guard !peerConnectionWrapper.isMCUPublisherPeer, !peerConnectionWrapper.isDummyPeer, + peerConnectionWrapper.roomType == kRoomTypeVideo + else { continue } + + peerConnectionWrapper.isAudioOnly = false + self.requestOfferWithRepetition(forSessionId: peerConnectionWrapper.peerId, withRoomType: kRoomTypeVideo, withSid: peerConnectionWrapper.sid) + } + + // Let the other participants know that we are now in the call with video + self.updateCallFlagsInServer() + self.startSendingCurrentState() + + if CallKitManager.isCallKitAvailable() { + CallKitManager.sharedInstance().changeHasVideo(true, forCall: self.room.token) + } + } + } + + public func downgradeToVoiceOnlyCall() { + WebRTCCommon.shared.dispatch { + guard !self.isAudioOnly, self.supportsCallUpgradeUsingRenegotiation else { return } + + NCLog.log("Downgrading video call to voice only call using renegotiation") + + self.isAudioOnly = true + self.disableVideoAtStart = true + + NCAudioController.shared.setAudioSessionToVoiceChatMode() + + // Stop sending the local video, but keep the negotiated m-line (a sender without + // a track), so the call can be upgraded to a video call again later + if let peerConnection = self.publisherPeerConnection?.getPeerConnection(), + let videoTransceiver = peerConnection.transceivers.first(where: { $0.mediaType == .video }) { + videoTransceiver.sender.track = nil + } + + // Stop capturing and release the local video track and the camera + self.cameraController?.stopAVCaptureSession() + self.cameraController = nil + self.localVideoTrack = nil + + // Stop receiving remote videos. Requesting an offer with the "sid" of the existing + // connection makes the MCU update that connection with a renegotiation offer, which + // is then answered with inactive video transceivers, since the call is audio only again. + for (_, peerConnectionWrapper) in self.connectionsDict { + guard !peerConnectionWrapper.isMCUPublisherPeer, !peerConnectionWrapper.isDummyPeer, + peerConnectionWrapper.roomType == kRoomTypeVideo + else { continue } + + peerConnectionWrapper.isAudioOnly = true + self.requestOfferWithRepetition(forSessionId: peerConnectionWrapper.peerId, withRoomType: kRoomTypeVideo, withSid: peerConnectionWrapper.sid) + } + + // Let the other participants know that we are now in the call without video + self.updateCallFlagsInServer() + self.startSendingCurrentState() + + if CallKitManager.isCallKitAvailable() { + CallKitManager.sharedInstance().changeHasVideo(false, forCall: self.room.token) + } + } + } + + private func updateCallFlagsInServer() { + NCAPIController.shared.updateCallFlags(inRoom: self.room.token, withCallFlags: self.joinCallFlags, forAccount: self.account) { error in + if let error { + NCLog.log("Could not update call flags. Error: \(error)") + } + } + } + public func enableVideo(_ enable: Bool) { WebRTCCommon.shared.dispatch { if enable { @@ -1025,6 +1145,15 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling if let localVideoTrack { peerConnection.add(localVideoTrack, streamIds: [NCCallController.kNCMediaStreamId]) + } else if self.room.canPublishVideo { + // In voice only calls, already negotiate a placeholder video m-line (a transceiver + // without a track), so the call can later be upgraded to a video call by just replacing + // the sender track. A renegotiation that adds a new m-line to an existing publisher + // peer connection completes on the SDP level, but the MCU never relays its media. + let transceiverInit = RTCRtpTransceiverInit() + transceiverInit.direction = .sendOnly + transceiverInit.streamIds = [NCCallController.kNCMediaStreamId] + peerConnection.addTransceiver(of: .video, init: transceiverInit) } peerConnectionWrapper.sendPublisherOffer() @@ -1060,24 +1189,49 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling peerConnectionWrapper.sendPublisherOffer() } - public func requestOfferWithRepetition(forSessionId sessionId: String, withRoomType roomType: String) { + public func requestOfferWithRepetition(forSessionId sessionId: String, withRoomType roomType: String, withSid sid: String? = nil) { WebRTCCommon.shared.assertQueue() + // Requesting an offer with a "sid" updates the existing connection. If that connection has + // failed it would require an ICE restart rather than an update to recover, so request a new + // connection instead (by omitting the "sid"). The offer will then arrive with a new "sid", + // which replaces the failed peer connection (see processOfferAnswer). + let isConnectionFailed = self.getPeerConnectionWrapper(forSessionId: sessionId, ofType: roomType)?.isConnectionFailed() ?? false + let updateSid = isConnectionFailed ? nil : sid + + let peerKey = self.getPeerKey(withSessionId: sessionId, ofType: roomType, forOwnScreenshare: false) + + if let previousOfferTimer = self.pendingOffersDict[peerKey], previousOfferTimer.isValid { + // If a connection needs to be updated ("sid" is set) but another offer request is already + // pending, ignore the new update. A new connection needs to be requested even if there + // is another one already pending. + if updateSid != nil { + return + } + + DispatchQueue.main.async { + previousOfferTimer.invalidate() + } + } + let timeout = Int(Date().timeIntervalSince1970 + 60) - let userInfo: [String: Any] = [ + var userInfo: [String: Any] = [ "sessionId": sessionId, "roomType": roomType, "timeout": timeout ] + if let updateSid { + userInfo["sid"] = updateSid + } + let pendingOfferTimer = Timer(timeInterval: 8.0, target: self, selector: #selector(self.requestNewOffer), userInfo: userInfo, repeats: true) - let peerKey = self.getPeerKey(withSessionId: sessionId, ofType: roomType, forOwnScreenshare: false) self.pendingOffersDict[peerKey] = pendingOfferTimer // Request new offer if let externalSignalingController { - externalSignalingController.requestOffer(forSessionId: sessionId, andRoomType: roomType) + externalSignalingController.requestOffer(forSessionId: sessionId, andRoomType: roomType, withSid: updateSid) } DispatchQueue.main.async { @@ -1093,10 +1247,12 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling let externalSignalingController else { return } + let sid = userInfo["sid"] as? String + WebRTCCommon.shared.dispatch { if Int(Date().timeIntervalSince1970) < timeout { print("Re-requesting an offer to session \(sessionId)") - externalSignalingController.requestOffer(forSessionId: sessionId, andRoomType: roomType) + externalSignalingController.requestOffer(forSessionId: sessionId, andRoomType: roomType, withSid: sid) } else { DispatchQueue.main.async { timer.invalidate() @@ -1323,6 +1479,8 @@ internal class NCCallController: NSObject, NCPeerConnectionDelegate, NCSignaling } func peerConnection(_ peerConnection: NCPeerConnection, didRemove stream: RTCMediaStream?) { + // The stream is nil when a receiver was removed (e.g. when a video transceiver + // was set to inactive in an audio only call) guard !peerConnection.isMCUPublisherPeer, let stream else { return } self.delegate?.callController(self, didRemoveStream: stream, ofPeer: peerConnection) diff --git a/NextcloudTalk/Network/NCAPIController.swift b/NextcloudTalk/Network/NCAPIController.swift index f217c7674..88c29cc2e 100644 --- a/NextcloudTalk/Network/NCAPIController.swift +++ b/NextcloudTalk/Network/NCAPIController.swift @@ -1924,6 +1924,19 @@ class NCAPIController: NSObject, NKCommonDelegate { } } + @discardableResult + public func updateCallFlags(inRoom token: String, withCallFlags flags: CallFlag, forAccount account: TalkAccount, completionBlock: @escaping (_ error: OcsError?) -> Void) -> URLSessionDataTask? { + guard let apiSessionManager = self.getAPISessionManager(forAccountId: account.accountId), + let encodedToken = token.addingPercentEncoding(withAllowedCharacters: .urlHostAllowed) + else { return nil } + + let urlString = self.getRequestURL(forEndpoint: "call/\(encodedToken)", withAPIType: .call, forAccount: account) + + return apiSessionManager.putOcs(urlString, account: account, parameters: ["flags": flags.rawValue]) { _, ocsError in + completionBlock(ocsError) + } + } + @discardableResult public func leaveCall(inRoom token: String, forAllParticipants allParticipants: Bool, forAccount account: TalkAccount, completionBlock: @escaping (_ error: OcsError?) -> Void) -> URLSessionDataTask? { guard let apiSessionManager = self.getAPISessionManager(forAccountId: account.accountId), diff --git a/NextcloudTalk/WebRTC/NCExternalSignalingController.swift b/NextcloudTalk/WebRTC/NCExternalSignalingController.swift index 1c1325dec..5ad72f843 100644 --- a/NextcloudTalk/WebRTC/NCExternalSignalingController.swift +++ b/NextcloudTalk/WebRTC/NCExternalSignalingController.swift @@ -46,6 +46,7 @@ public enum NCExternalSignalingSendMessageStatus { public private(set) var account: TalkAccount public private(set) var disconnected: Bool = true public private(set) var hasMCU: Bool = false + public private(set) var hasUpdateSdp: Bool = false public private(set) var hasChatRelay: Bool = false public private(set) var sessionId: String? public private(set) var participantsMap = [String: SignalingParticipant]() @@ -348,6 +349,7 @@ public enum NCExternalSignalingSendMessageStatus { } self.hasMCU = serverFeatures.contains(where: { $0 == "mcu" }) + self.hasUpdateSdp = serverFeatures.contains(where: { $0 == "update-sdp" }) self.hasChatRelay = serverFeatures.contains(where: { $0 == "chat-relay" }) DispatchQueue.main.async { @@ -499,7 +501,18 @@ public enum NCExternalSignalingSendMessageStatus { self.send(message: messageDict, withCompletionBlock: nil) } - func requestOffer(forSessionId sessionId: String, andRoomType roomType: String) { + func requestOffer(forSessionId sessionId: String, andRoomType roomType: String, withSid sid: String? = nil) { + // When a "sid" of an existing connection is provided (and the server supports "update-sdp"), + // the MCU updates that connection with a renegotiation offer instead of creating a new one + var dataDict: [AnyHashable: Any] = [ + "type": "requestoffer", + "roomType": roomType + ] + + if let sid { + dataDict["sid"] = sid + } + let messageDict: [AnyHashable: Any] = [ "type": "message", "message": [ @@ -507,10 +520,7 @@ public enum NCExternalSignalingSendMessageStatus { "type": "session", "sessionid": sessionId ], - "data": [ - "type": "requestoffer", - "roomType": roomType - ] + "data": dataDict ] ] diff --git a/NextcloudTalk/WebRTC/NCPeerConnection.swift b/NextcloudTalk/WebRTC/NCPeerConnection.swift index 732f6fedf..8c2a90925 100644 --- a/NextcloudTalk/WebRTC/NCPeerConnection.swift +++ b/NextcloudTalk/WebRTC/NCPeerConnection.swift @@ -176,10 +176,13 @@ public class NCPeerConnection: NSObject { WebRTCCommon.shared.assertQueue() // Create data channel before creating the offer to enable data channels - let config = RTCDataChannelConfiguration() - config.isNegotiated = false - localDataChannel = peerConnection?.dataChannel(forLabel: "status", configuration: config) - localDataChannel?.delegate = self + // Skip if it already exists, when sending a renegotiation offer on an established connection + if localDataChannel == nil { + let config = RTCDataChannelConfiguration() + config.isNegotiated = false + localDataChannel = peerConnection?.dataChannel(forLabel: "status", configuration: config) + localDataChannel?.delegate = self + } // Create offer peerConnection?.offer(for: constraints) { [weak self] sdp, error in @@ -247,6 +250,14 @@ public class NCPeerConnection: NSObject { return peerConnection } + func isConnectionFailed() -> Bool { + WebRTCCommon.shared.assertQueue() + + guard let peerConnection else { return false } + + return peerConnection.iceConnectionState == .failed || peerConnection.connectionState == .failed + } + func getLocalDataChannel() -> RTCDataChannel? { WebRTCCommon.shared.assertQueue() return localDataChannel @@ -353,17 +364,27 @@ public class NCPeerConnection: NSObject { // If we just set a remote offer we need to create an answer and set it as local description. if peerConnection?.signalingState == .haveRemoteOffer { // Create data channel before sending answer - let config = RTCDataChannelConfiguration() - config.isNegotiated = false - localDataChannel = peerConnection?.dataChannel(forLabel: "status", configuration: config) - localDataChannel?.delegate = self + // Skip if it already exists, when answering a renegotiation offer on an established connection + if localDataChannel == nil { + let config = RTCDataChannelConfiguration() + config.isNegotiated = false + localDataChannel = peerConnection?.dataChannel(forLabel: "status", configuration: config) + localDataChannel?.delegate = self + } - // Stop video transceiver in audio only peer connections + // Decline to receive video in audio only peer connections by setting the video transceivers + // to "inactive". The m-line needs to stay negotiated (so it must not be rejected by stopping + // the transceiver), otherwise it can not be activated again when the call is upgraded to a + // video call later. Reactivating an "inactive" transceiver is the only kind of renegotiation + // that is known to work with the MCU (it is what the web client does to block remote videos). // Constraints are no longer supported when creating answers (with Unified Plan semantics) - if isAudioOnly { - for transceiver in peerConnection?.transceivers ?? [] where transceiver.mediaType == .video { - transceiver.stopInternal() - NSLog("Stop video transceiver in audio only peer connections.") + for transceiver in peerConnection?.transceivers ?? [] where transceiver.mediaType == .video { + if isAudioOnly, transceiver.direction != .inactive { + NSLog("Set video transceiver to inactive in audio only peer connection.") + transceiver.setDirection(.inactive, error: nil) + } else if !isAudioOnly, transceiver.direction == .inactive { + NSLog("Reactivate inactive video transceiver to receive video again.") + transceiver.setDirection(.recvOnly, error: nil) } } @@ -517,7 +538,10 @@ extension NCPeerConnection: RTCPeerConnectionDelegate { } public func peerConnectionShouldNegotiate(_ peerConnection: RTCPeerConnection) { - NSLog("WARNING: Renegotiation needed but unimplemented.") + // Client initiated renegotiation is not supported. Offers are created explicitly and + // tracks are only added to or removed from already negotiated m-lines (replaceTrack), + // which does not require a renegotiation. + NSLog("Negotiation needed for peer '%@' -> ignoring", peerId) } public func peerConnection(_ peerConnection: RTCPeerConnection, didChange newState: RTCIceConnectionState) { diff --git a/NextcloudTalk/en.lproj/Localizable.strings b/NextcloudTalk/en.lproj/Localizable.strings index 979f81e61..b98f914e0 100644 --- a/NextcloudTalk/en.lproj/Localizable.strings +++ b/NextcloudTalk/en.lproj/Localizable.strings @@ -2282,6 +2282,9 @@ /* No comment provided by engineer. */ "Switch account" = "Switch account"; +/* No comment provided by engineer. */ +"Switch to voice-only call" = "Switch to voice-only call"; + /* No comment provided by engineer. */ "Switching to another conversation …" = "Switching to another conversation …";