diff --git a/apps/ios/Shared/Views/Call/ActiveCallView.swift b/apps/ios/Shared/Views/Call/ActiveCallView.swift index ad9d90c38..650e9450a 100644 --- a/apps/ios/Shared/Views/Call/ActiveCallView.swift +++ b/apps/ios/Shared/Views/Call/ActiveCallView.swift @@ -52,7 +52,12 @@ struct ActiveCallView: View { AppDelegate.keepScreenOn(false) client?.endCall() } - .onChange(of: m.callCommand) { _ in sendCommandToClient()} + .onChange(of: m.callCommand) { cmd in + if let cmd = cmd { + m.callCommand = nil + sendCommandToClient(cmd) + } + } .background(.black) .preferredColorScheme(.dark) } @@ -60,16 +65,17 @@ struct ActiveCallView: View { private func createWebRTCClient() { if client == nil && canConnectCall { client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio) - sendCommandToClient() + if let cmd = m.callCommand { + m.callCommand = nil + sendCommandToClient(cmd) + } } } - private func sendCommandToClient() { + private func sendCommandToClient(_ cmd: WCallCommand) { if call == m.activeCall, m.activeCall != nil, - let client = client, - let cmd = m.callCommand { - m.callCommand = nil + let client = client { logger.debug("sendCallCommand: \(cmd.cmdType)") Task { await client.sendCallCommand(command: cmd) @@ -255,7 +261,6 @@ struct ActiveCallOverlay: View { HStack { Text(call.encryptionStatus) if let connInfo = call.connectionInfo { -// Text("(") + Text(connInfo.text) + Text(", \(connInfo.protocolText))") Text("(") + Text(connInfo.text) + Text(")") } } diff --git a/apps/ios/Shared/Views/Call/CallManager.swift b/apps/ios/Shared/Views/Call/CallManager.swift index 6e3066d1a..6b71e88cf 100644 --- a/apps/ios/Shared/Views/Call/CallManager.swift +++ b/apps/ios/Shared/Views/Call/CallManager.swift @@ -94,6 +94,8 @@ class CallManager { completed() } else { logger.debug("CallManager.endCall: ending call...") + // TODO this command won't be executed because activeCall is assigned nil, + // and there is a condition in sendCommandToClient that would prevent its execution. m.callCommand = .end m.activeCall = nil m.showCallView = false diff --git a/apps/ios/Shared/Views/Call/WebRTC.swift b/apps/ios/Shared/Views/Call/WebRTC.swift index ceeaf513d..a36a239e2 100644 --- a/apps/ios/Shared/Views/Call/WebRTC.swift +++ b/apps/ios/Shared/Views/Call/WebRTC.swift @@ -358,26 +358,12 @@ struct ConnectionInfo: Codable, Equatable { return "\(local?.rawValue ?? unknown) / \(remote?.rawValue ?? unknown)" } } - - var protocolText: String { - let unknown = NSLocalizedString("unknown", comment: "connection info") - let local = localCandidate?.protocol?.uppercased() ?? unknown - let localRelay = localCandidate?.relayProtocol?.uppercased() ?? unknown - let remote = remoteCandidate?.protocol?.uppercased() ?? unknown - let localText = localRelay == local || localCandidate?.relayProtocol == nil - ? local - : "\(local) (\(localRelay))" - return local == remote - ? localText - : "\(localText) / \(remote)" - } } // https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate struct RTCIceCandidate: Codable, Equatable { var candidateType: RTCIceCandidateType? var `protocol`: String? - var relayProtocol: String? var sdpMid: String? var sdpMLineIndex: Int? var candidate: String diff --git a/apps/ios/Shared/Views/Call/WebRTCClient.swift b/apps/ios/Shared/Views/Call/WebRTCClient.swift index 5ecec1a98..b3cad62fa 100644 --- a/apps/ios/Shared/Views/Call/WebRTCClient.swift +++ b/apps/ios/Shared/Views/Call/WebRTCClient.swift @@ -21,7 +21,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg struct Call { var connection: RTCPeerConnection - var iceCandidates: [RTCIceCandidate] + var iceCandidates: IceCandidates var localMedia: CallMediaType var localCamera: RTCVideoCapturer? var localVideoSource: RTCVideoSource? @@ -33,6 +33,20 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg var frameDecryptor: RTCFrameDecryptor? } + actor IceCandidates { + private var candidates: [RTCIceCandidate] = [] + + func getAndClear() async -> [RTCIceCandidate] { + let cs = candidates + candidates = [] + return cs + } + + func append(_ c: RTCIceCandidate) async { + candidates.append(c) + } + } + private let rtcAudioSession = RTCAudioSession.sharedInstance() private let audioQueue = DispatchQueue(label: "audio") private var sendCallResponse: (WVAPIMessage) async -> Void @@ -60,7 +74,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg WebRTC.RTCIceServer(urlStrings: ["turn:turn.simplex.im:443?transport=tcp"], username: "private", credential: "yleob6AVkiNI87hpR94Z"), ] - func initializeCall(_ iceServers: [WebRTC.RTCIceServer]?, _ remoteIceCandidates: [RTCIceCandidate], _ mediaType: CallMediaType, _ aesKey: String?, _ relay: Bool?) -> Call { + func initializeCall(_ iceServers: [WebRTC.RTCIceServer]?, _ mediaType: CallMediaType, _ aesKey: String?, _ relay: Bool?) -> Call { let connection = createPeerConnection(iceServers ?? getWebRTCIceServers() ?? defaultIceServers, relay) connection.delegate = self createAudioSender(connection) @@ -87,7 +101,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg } return Call( connection: connection, - iceCandidates: remoteIceCandidates, + iceCandidates: IceCandidates(), localMedia: mediaType, localCamera: localCamera, localVideoSource: localVideoSource, @@ -144,26 +158,21 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg logger.debug("starting incoming call - create webrtc session") if activeCall.wrappedValue != nil { endCall() } let encryption = WebRTCClient.enableEncryption - let call = initializeCall(iceServers?.toWebRTCIceServers(), [], media, encryption ? aesKey : nil, relay) + let call = initializeCall(iceServers?.toWebRTCIceServers(), media, encryption ? aesKey : nil, relay) activeCall.wrappedValue = call call.connection.offer { answer in Task { - let gotCandidates = await self.waitWithTimeout(10_000, stepMs: 1000, until: { self.activeCall.wrappedValue?.iceCandidates.count ?? 0 > 0 }) - if gotCandidates { - await self.sendCallResponse(.init( - corrId: nil, - resp: .offer( - offer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))), - iceCandidates: compressToBase64(input: encodeJSON(self.activeCall.wrappedValue?.iceCandidates ?? [])), - capabilities: CallCapabilities(encryption: encryption) - ), - command: command) - ) - } else { - self.endCall() - } + await self.sendCallResponse(.init( + corrId: nil, + resp: .offer( + offer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))), + iceCandidates: compressToBase64(input: encodeJSON(await self.getInitialIceCandidates())), + capabilities: CallCapabilities(encryption: encryption) + ), + command: command) + ) + await self.waitForMoreIceCandidates() } - } case let .offer(offer, iceCandidates, media, aesKey, iceServers, relay): if activeCall.wrappedValue != nil { @@ -172,7 +181,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg resp = .error(message: "accept: encryption is not supported") } else if let offer: CustomRTCSessionDescription = decodeJSON(decompressFromBase64(input: offer)), let remoteIceCandidates: [RTCIceCandidate] = decodeJSON(decompressFromBase64(input: iceCandidates)) { - let call = initializeCall(iceServers?.toWebRTCIceServers(), remoteIceCandidates, media, WebRTCClient.enableEncryption ? aesKey : nil, relay) + let call = initializeCall(iceServers?.toWebRTCIceServers(), media, WebRTCClient.enableEncryption ? aesKey : nil, relay) activeCall.wrappedValue = call let pc = call.connection if let type = offer.type, let sdp = offer.sdp { @@ -186,10 +195,11 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg corrId: nil, resp: .answer( answer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))), - iceCandidates: compressToBase64(input: encodeJSON(call.iceCandidates)) + iceCandidates: compressToBase64(input: encodeJSON(await self.getInitialIceCandidates())) ), command: command) ) + await self.waitForMoreIceCandidates() } // } } @@ -234,6 +244,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg resp = .ok } case .end: + // TODO possibly, endCall should be called before returning .ok await sendCallResponse(.init(corrId: nil, resp: .ok, command: command)) endCall() } @@ -242,6 +253,31 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg } } + func getInitialIceCandidates() async -> [RTCIceCandidate] { + await untilIceComplete(timeoutMs: 750, stepMs: 150) {} + let candidates = await activeCall.wrappedValue?.iceCandidates.getAndClear() ?? [] + logger.debug("WebRTCClient: sending initial ice candidates: \(candidates.count)") + return candidates + } + + func waitForMoreIceCandidates() async { + await untilIceComplete(timeoutMs: 12000, stepMs: 1500) { + let candidates = await self.activeCall.wrappedValue?.iceCandidates.getAndClear() ?? [] + if candidates.count > 0 { + logger.debug("WebRTCClient: sending more ice candidates: \(candidates.count)") + await self.sendIceCandidates(candidates) + } + } + } + + func sendIceCandidates(_ candidates: [RTCIceCandidate]) async { + await self.sendCallResponse(.init( + corrId: nil, + resp: .ice(iceCandidates: compressToBase64(input: encodeJSON(candidates))), + command: nil) + ) + } + func enableMedia(_ media: CallMediaType, _ enable: Bool) { logger.debug("WebRTCClient: enabling media \(media.rawValue) \(enable)") media == .video ? setVideoEnabled(enable) : setAudioEnabled(enable) @@ -387,12 +423,13 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg audioSessionToDefaults() } - func waitWithTimeout(_ timeoutMs: UInt64, stepMs: UInt64, until success: () -> Bool) async -> Bool { - let startedAt = DispatchTime.now() - while !success() && startedAt.uptimeNanoseconds + timeoutMs * 1000000 > DispatchTime.now().uptimeNanoseconds { - guard let _ = try? await Task.sleep(nanoseconds: stepMs * 1000000) else { break } - } - return success() + func untilIceComplete(timeoutMs: UInt64, stepMs: UInt64, action: @escaping () async -> Void) async { + var t: UInt64 = 0 + repeat { + _ = try? await Task.sleep(nanoseconds: stepMs * 1000000) + t += stepMs + await action() + } while t < timeoutMs && activeCall.wrappedValue?.connection.iceGatheringState != .complete } } @@ -479,6 +516,7 @@ extension WebRTCClient: RTCPeerConnectionDelegate { default: enableSpeaker = false } setSpeakerEnabledAndConfigureSession(enableSpeaker) + case .connected: sendConnectedEvent(connection) case .disconnected, .failed: endCall() default: do {} } @@ -491,7 +529,9 @@ extension WebRTCClient: RTCPeerConnectionDelegate { func peerConnection(_ connection: RTCPeerConnection, didGenerate candidate: WebRTC.RTCIceCandidate) { // logger.debug("Connection generated candidate \(candidate.debugDescription)") - activeCall.wrappedValue?.iceCandidates.append(candidate.toCandidate(nil, nil, nil)) + Task { + await self.activeCall.wrappedValue?.iceCandidates.append(candidate.toCandidate(nil, nil)) + } } func peerConnection(_ connection: RTCPeerConnection, didRemove candidates: [WebRTC.RTCIceCandidate]) { @@ -506,10 +546,9 @@ extension WebRTCClient: RTCPeerConnectionDelegate { lastReceivedMs lastDataReceivedMs: Int32, changeReason reason: String) { // logger.debug("Connection changed candidate \(reason) \(remote.debugDescription) \(remote.description)") - sendConnectedEvent(connection, local: local, remote: remote) } - func sendConnectedEvent(_ connection: WebRTC.RTCPeerConnection, local: WebRTC.RTCIceCandidate, remote: WebRTC.RTCIceCandidate) { + func sendConnectedEvent(_ connection: WebRTC.RTCPeerConnection) { connection.statistics { (stats: RTCStatisticsReport) in stats.statistics.values.forEach { stat in // logger.debug("Stat \(stat.debugDescription)") @@ -517,24 +556,25 @@ extension WebRTCClient: RTCPeerConnectionDelegate { let localId = stat.values["localCandidateId"] as? String, let remoteId = stat.values["remoteCandidateId"] as? String, let localStats = stats.statistics[localId], - let remoteStats = stats.statistics[remoteId], - local.sdp.contains("\((localStats.values["ip"] as? String ?? "--")) \((localStats.values["port"] as? String ?? "--"))") && - remote.sdp.contains("\((remoteStats.values["ip"] as? String ?? "--")) \((remoteStats.values["port"] as? String ?? "--"))") + let remoteStats = stats.statistics[remoteId] { Task { await self.sendCallResponse(.init( corrId: nil, resp: .connected(connectionInfo: ConnectionInfo( - localCandidate: local.toCandidate( - RTCIceCandidateType.init(rawValue: localStats.values["candidateType"] as! String), - localStats.values["protocol"] as? String, - localStats.values["relayProtocol"] as? String + localCandidate: RTCIceCandidate( + candidateType: RTCIceCandidateType.init(rawValue: localStats.values["candidateType"] as! String), + protocol: localStats.values["protocol"] as? String, + sdpMid: nil, + sdpMLineIndex: nil, + candidate: "" ), - remoteCandidate: remote.toCandidate( - RTCIceCandidateType.init(rawValue: remoteStats.values["candidateType"] as! String), - remoteStats.values["protocol"] as? String, - remoteStats.values["relayProtocol"] as? String - ))), + remoteCandidate: RTCIceCandidate( + candidateType: RTCIceCandidateType.init(rawValue: remoteStats.values["candidateType"] as! String), + protocol: remoteStats.values["protocol"] as? String, + sdpMid: nil, + sdpMLineIndex: nil, + candidate: ""))), command: nil) ) } @@ -634,11 +674,10 @@ extension RTCIceCandidate { } extension WebRTC.RTCIceCandidate { - func toCandidate(_ candidateType: RTCIceCandidateType?, _ protocol: String?, _ relayProtocol: String?) -> RTCIceCandidate { + func toCandidate(_ candidateType: RTCIceCandidateType?, _ protocol: String?) -> RTCIceCandidate { RTCIceCandidate( candidateType: candidateType, protocol: `protocol`, - relayProtocol: relayProtocol, sdpMid: sdpMid, sdpMLineIndex: Int(sdpMLineIndex), candidate: sdp diff --git a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt index 51c362325..5f30d21bb 100644 --- a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt +++ b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt @@ -370,7 +370,6 @@ fun CallInfoView(call: Call, alignment: Alignment.Horizontal) { InfoText(call.callState.text) val connInfo = call.connectionInfo - // val connInfoText = if (connInfo == null) "" else " (${connInfo.text}, ${connInfo.protocolText})" val connInfoText = if (connInfo == null) "" else " (${connInfo.text})" InfoText(call.encryptionStatus + connInfoText) } @@ -585,8 +584,8 @@ fun PreviewActiveCallOverlayVideo() { localMedia = CallMediaType.Video, peerMedia = CallMediaType.Video, connectionInfo = ConnectionInfo( - RTCIceCandidate(RTCIceCandidateType.Host, "tcp", null), - RTCIceCandidate(RTCIceCandidateType.Host, "tcp", null) + RTCIceCandidate(RTCIceCandidateType.Host, "tcp"), + RTCIceCandidate(RTCIceCandidateType.Host, "tcp") ) ), speakerCanBeEnabled = true, @@ -611,8 +610,8 @@ fun PreviewActiveCallOverlayAudio() { localMedia = CallMediaType.Audio, peerMedia = CallMediaType.Audio, connectionInfo = ConnectionInfo( - RTCIceCandidate(RTCIceCandidateType.Host, "udp", null), - RTCIceCandidate(RTCIceCandidateType.Host, "udp", null) + RTCIceCandidate(RTCIceCandidateType.Host, "udp"), + RTCIceCandidate(RTCIceCandidateType.Host, "udp") ) ), speakerCanBeEnabled = true, diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt index 6a357d26f..3e79dfb4f 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt @@ -127,18 +127,10 @@ sealed class WCallResponse { "${local?.value ?: "unknown"} / ${remote?.value ?: "unknown"}" } } - - val protocolText: String get() { - val local = localCandidate?.protocol?.uppercase(Locale.ROOT) ?: "unknown" - val localRelay = localCandidate?.relayProtocol?.uppercase(Locale.ROOT) ?: "unknown" - val remote = remoteCandidate?.protocol?.uppercase(Locale.ROOT) ?: "unknown" - val localText = if (localRelay == local || localCandidate?.relayProtocol == null) local else "$local ($localRelay)" - return if (local == remote) localText else "$localText / $remote" - } } // https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate -@Serializable data class RTCIceCandidate(val candidateType: RTCIceCandidateType?, val protocol: String?, val relayProtocol: String?) +@Serializable data class RTCIceCandidate(val candidateType: RTCIceCandidateType?, val protocol: String?) // https://developer.mozilla.org/en-US/docs/Web/API/RTCIceServer @Serializable data class RTCIceServer(val urls: List, val username: String? = null, val credential: String? = null) diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt index c2665109f..22d39409e 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt @@ -136,7 +136,6 @@ private fun SendStateUpdates() { .collect { call -> val state = call.callState.text val connInfo = call.connectionInfo - // val connInfoText = if (connInfo == null) "" else " (${connInfo.text}, ${connInfo.protocolText})" val connInfoText = if (connInfo == null) "" else " (${connInfo.text})" val description = call.encryptionStatus + connInfoText chatModel.callCommand.add(WCallCommand.Description(state, description))