ios: fix calls connecting state (#3475)

* ios: fix calls connecting state

* optimization

* changes

* removed relay protocol

* simplify

* use actor

* fix loop, better onChange, some questions

* remove extra iteration

---------

Co-authored-by: Avently <avently@local>
Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com>
This commit is contained in:
Stanislav Dmitrenko 2023-11-28 06:20:51 +08:00 committed by GitHub
parent 05278e5a06
commit 950bbe19da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 102 additions and 80 deletions

View File

@ -52,7 +52,12 @@ struct ActiveCallView: View {
AppDelegate.keepScreenOn(false) AppDelegate.keepScreenOn(false)
client?.endCall() client?.endCall()
} }
.onChange(of: m.callCommand) { _ in sendCommandToClient()} .onChange(of: m.callCommand) { cmd in
if let cmd = cmd {
m.callCommand = nil
sendCommandToClient(cmd)
}
}
.background(.black) .background(.black)
.preferredColorScheme(.dark) .preferredColorScheme(.dark)
} }
@ -60,16 +65,17 @@ struct ActiveCallView: View {
private func createWebRTCClient() { private func createWebRTCClient() {
if client == nil && canConnectCall { if client == nil && canConnectCall {
client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio) client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio)
sendCommandToClient() if let cmd = m.callCommand {
m.callCommand = nil
sendCommandToClient(cmd)
}
} }
} }
private func sendCommandToClient() { private func sendCommandToClient(_ cmd: WCallCommand) {
if call == m.activeCall, if call == m.activeCall,
m.activeCall != nil, m.activeCall != nil,
let client = client, let client = client {
let cmd = m.callCommand {
m.callCommand = nil
logger.debug("sendCallCommand: \(cmd.cmdType)") logger.debug("sendCallCommand: \(cmd.cmdType)")
Task { Task {
await client.sendCallCommand(command: cmd) await client.sendCallCommand(command: cmd)
@ -255,7 +261,6 @@ struct ActiveCallOverlay: View {
HStack { HStack {
Text(call.encryptionStatus) Text(call.encryptionStatus)
if let connInfo = call.connectionInfo { if let connInfo = call.connectionInfo {
// Text("(") + Text(connInfo.text) + Text(", \(connInfo.protocolText))")
Text("(") + Text(connInfo.text) + Text(")") Text("(") + Text(connInfo.text) + Text(")")
} }
} }

View File

@ -94,6 +94,8 @@ class CallManager {
completed() completed()
} else { } else {
logger.debug("CallManager.endCall: ending call...") logger.debug("CallManager.endCall: ending call...")
// TODO this command won't be executed because activeCall is assigned nil,
// and there is a condition in sendCommandToClient that would prevent its execution.
m.callCommand = .end m.callCommand = .end
m.activeCall = nil m.activeCall = nil
m.showCallView = false m.showCallView = false

View File

@ -358,26 +358,12 @@ struct ConnectionInfo: Codable, Equatable {
return "\(local?.rawValue ?? unknown) / \(remote?.rawValue ?? unknown)" return "\(local?.rawValue ?? unknown) / \(remote?.rawValue ?? unknown)"
} }
} }
var protocolText: String {
let unknown = NSLocalizedString("unknown", comment: "connection info")
let local = localCandidate?.protocol?.uppercased() ?? unknown
let localRelay = localCandidate?.relayProtocol?.uppercased() ?? unknown
let remote = remoteCandidate?.protocol?.uppercased() ?? unknown
let localText = localRelay == local || localCandidate?.relayProtocol == nil
? local
: "\(local) (\(localRelay))"
return local == remote
? localText
: "\(localText) / \(remote)"
}
} }
// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate // https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate
struct RTCIceCandidate: Codable, Equatable { struct RTCIceCandidate: Codable, Equatable {
var candidateType: RTCIceCandidateType? var candidateType: RTCIceCandidateType?
var `protocol`: String? var `protocol`: String?
var relayProtocol: String?
var sdpMid: String? var sdpMid: String?
var sdpMLineIndex: Int? var sdpMLineIndex: Int?
var candidate: String var candidate: String

View File

@ -21,7 +21,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
struct Call { struct Call {
var connection: RTCPeerConnection var connection: RTCPeerConnection
var iceCandidates: [RTCIceCandidate] var iceCandidates: IceCandidates
var localMedia: CallMediaType var localMedia: CallMediaType
var localCamera: RTCVideoCapturer? var localCamera: RTCVideoCapturer?
var localVideoSource: RTCVideoSource? var localVideoSource: RTCVideoSource?
@ -33,6 +33,20 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
var frameDecryptor: RTCFrameDecryptor? var frameDecryptor: RTCFrameDecryptor?
} }
actor IceCandidates {
private var candidates: [RTCIceCandidate] = []
func getAndClear() async -> [RTCIceCandidate] {
let cs = candidates
candidates = []
return cs
}
func append(_ c: RTCIceCandidate) async {
candidates.append(c)
}
}
private let rtcAudioSession = RTCAudioSession.sharedInstance() private let rtcAudioSession = RTCAudioSession.sharedInstance()
private let audioQueue = DispatchQueue(label: "audio") private let audioQueue = DispatchQueue(label: "audio")
private var sendCallResponse: (WVAPIMessage) async -> Void private var sendCallResponse: (WVAPIMessage) async -> Void
@ -60,7 +74,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
WebRTC.RTCIceServer(urlStrings: ["turn:turn.simplex.im:443?transport=tcp"], username: "private", credential: "yleob6AVkiNI87hpR94Z"), WebRTC.RTCIceServer(urlStrings: ["turn:turn.simplex.im:443?transport=tcp"], username: "private", credential: "yleob6AVkiNI87hpR94Z"),
] ]
func initializeCall(_ iceServers: [WebRTC.RTCIceServer]?, _ remoteIceCandidates: [RTCIceCandidate], _ mediaType: CallMediaType, _ aesKey: String?, _ relay: Bool?) -> Call { func initializeCall(_ iceServers: [WebRTC.RTCIceServer]?, _ mediaType: CallMediaType, _ aesKey: String?, _ relay: Bool?) -> Call {
let connection = createPeerConnection(iceServers ?? getWebRTCIceServers() ?? defaultIceServers, relay) let connection = createPeerConnection(iceServers ?? getWebRTCIceServers() ?? defaultIceServers, relay)
connection.delegate = self connection.delegate = self
createAudioSender(connection) createAudioSender(connection)
@ -87,7 +101,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
} }
return Call( return Call(
connection: connection, connection: connection,
iceCandidates: remoteIceCandidates, iceCandidates: IceCandidates(),
localMedia: mediaType, localMedia: mediaType,
localCamera: localCamera, localCamera: localCamera,
localVideoSource: localVideoSource, localVideoSource: localVideoSource,
@ -144,26 +158,21 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
logger.debug("starting incoming call - create webrtc session") logger.debug("starting incoming call - create webrtc session")
if activeCall.wrappedValue != nil { endCall() } if activeCall.wrappedValue != nil { endCall() }
let encryption = WebRTCClient.enableEncryption let encryption = WebRTCClient.enableEncryption
let call = initializeCall(iceServers?.toWebRTCIceServers(), [], media, encryption ? aesKey : nil, relay) let call = initializeCall(iceServers?.toWebRTCIceServers(), media, encryption ? aesKey : nil, relay)
activeCall.wrappedValue = call activeCall.wrappedValue = call
call.connection.offer { answer in call.connection.offer { answer in
Task { Task {
let gotCandidates = await self.waitWithTimeout(10_000, stepMs: 1000, until: { self.activeCall.wrappedValue?.iceCandidates.count ?? 0 > 0 }) await self.sendCallResponse(.init(
if gotCandidates { corrId: nil,
await self.sendCallResponse(.init( resp: .offer(
corrId: nil, offer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))),
resp: .offer( iceCandidates: compressToBase64(input: encodeJSON(await self.getInitialIceCandidates())),
offer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))), capabilities: CallCapabilities(encryption: encryption)
iceCandidates: compressToBase64(input: encodeJSON(self.activeCall.wrappedValue?.iceCandidates ?? [])), ),
capabilities: CallCapabilities(encryption: encryption) command: command)
), )
command: command) await self.waitForMoreIceCandidates()
)
} else {
self.endCall()
}
} }
} }
case let .offer(offer, iceCandidates, media, aesKey, iceServers, relay): case let .offer(offer, iceCandidates, media, aesKey, iceServers, relay):
if activeCall.wrappedValue != nil { if activeCall.wrappedValue != nil {
@ -172,7 +181,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
resp = .error(message: "accept: encryption is not supported") resp = .error(message: "accept: encryption is not supported")
} else if let offer: CustomRTCSessionDescription = decodeJSON(decompressFromBase64(input: offer)), } else if let offer: CustomRTCSessionDescription = decodeJSON(decompressFromBase64(input: offer)),
let remoteIceCandidates: [RTCIceCandidate] = decodeJSON(decompressFromBase64(input: iceCandidates)) { let remoteIceCandidates: [RTCIceCandidate] = decodeJSON(decompressFromBase64(input: iceCandidates)) {
let call = initializeCall(iceServers?.toWebRTCIceServers(), remoteIceCandidates, media, WebRTCClient.enableEncryption ? aesKey : nil, relay) let call = initializeCall(iceServers?.toWebRTCIceServers(), media, WebRTCClient.enableEncryption ? aesKey : nil, relay)
activeCall.wrappedValue = call activeCall.wrappedValue = call
let pc = call.connection let pc = call.connection
if let type = offer.type, let sdp = offer.sdp { if let type = offer.type, let sdp = offer.sdp {
@ -186,10 +195,11 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
corrId: nil, corrId: nil,
resp: .answer( resp: .answer(
answer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))), answer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))),
iceCandidates: compressToBase64(input: encodeJSON(call.iceCandidates)) iceCandidates: compressToBase64(input: encodeJSON(await self.getInitialIceCandidates()))
), ),
command: command) command: command)
) )
await self.waitForMoreIceCandidates()
} }
// } // }
} }
@ -234,6 +244,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
resp = .ok resp = .ok
} }
case .end: case .end:
// TODO possibly, endCall should be called before returning .ok
await sendCallResponse(.init(corrId: nil, resp: .ok, command: command)) await sendCallResponse(.init(corrId: nil, resp: .ok, command: command))
endCall() endCall()
} }
@ -242,6 +253,31 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
} }
} }
func getInitialIceCandidates() async -> [RTCIceCandidate] {
await untilIceComplete(timeoutMs: 750, stepMs: 150) {}
let candidates = await activeCall.wrappedValue?.iceCandidates.getAndClear() ?? []
logger.debug("WebRTCClient: sending initial ice candidates: \(candidates.count)")
return candidates
}
func waitForMoreIceCandidates() async {
await untilIceComplete(timeoutMs: 12000, stepMs: 1500) {
let candidates = await self.activeCall.wrappedValue?.iceCandidates.getAndClear() ?? []
if candidates.count > 0 {
logger.debug("WebRTCClient: sending more ice candidates: \(candidates.count)")
await self.sendIceCandidates(candidates)
}
}
}
func sendIceCandidates(_ candidates: [RTCIceCandidate]) async {
await self.sendCallResponse(.init(
corrId: nil,
resp: .ice(iceCandidates: compressToBase64(input: encodeJSON(candidates))),
command: nil)
)
}
func enableMedia(_ media: CallMediaType, _ enable: Bool) { func enableMedia(_ media: CallMediaType, _ enable: Bool) {
logger.debug("WebRTCClient: enabling media \(media.rawValue) \(enable)") logger.debug("WebRTCClient: enabling media \(media.rawValue) \(enable)")
media == .video ? setVideoEnabled(enable) : setAudioEnabled(enable) media == .video ? setVideoEnabled(enable) : setAudioEnabled(enable)
@ -387,12 +423,13 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
audioSessionToDefaults() audioSessionToDefaults()
} }
func waitWithTimeout(_ timeoutMs: UInt64, stepMs: UInt64, until success: () -> Bool) async -> Bool { func untilIceComplete(timeoutMs: UInt64, stepMs: UInt64, action: @escaping () async -> Void) async {
let startedAt = DispatchTime.now() var t: UInt64 = 0
while !success() && startedAt.uptimeNanoseconds + timeoutMs * 1000000 > DispatchTime.now().uptimeNanoseconds { repeat {
guard let _ = try? await Task.sleep(nanoseconds: stepMs * 1000000) else { break } _ = try? await Task.sleep(nanoseconds: stepMs * 1000000)
} t += stepMs
return success() await action()
} while t < timeoutMs && activeCall.wrappedValue?.connection.iceGatheringState != .complete
} }
} }
@ -479,6 +516,7 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
default: enableSpeaker = false default: enableSpeaker = false
} }
setSpeakerEnabledAndConfigureSession(enableSpeaker) setSpeakerEnabledAndConfigureSession(enableSpeaker)
case .connected: sendConnectedEvent(connection)
case .disconnected, .failed: endCall() case .disconnected, .failed: endCall()
default: do {} default: do {}
} }
@ -491,7 +529,9 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
func peerConnection(_ connection: RTCPeerConnection, didGenerate candidate: WebRTC.RTCIceCandidate) { func peerConnection(_ connection: RTCPeerConnection, didGenerate candidate: WebRTC.RTCIceCandidate) {
// logger.debug("Connection generated candidate \(candidate.debugDescription)") // logger.debug("Connection generated candidate \(candidate.debugDescription)")
activeCall.wrappedValue?.iceCandidates.append(candidate.toCandidate(nil, nil, nil)) Task {
await self.activeCall.wrappedValue?.iceCandidates.append(candidate.toCandidate(nil, nil))
}
} }
func peerConnection(_ connection: RTCPeerConnection, didRemove candidates: [WebRTC.RTCIceCandidate]) { func peerConnection(_ connection: RTCPeerConnection, didRemove candidates: [WebRTC.RTCIceCandidate]) {
@ -506,10 +546,9 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
lastReceivedMs lastDataReceivedMs: Int32, lastReceivedMs lastDataReceivedMs: Int32,
changeReason reason: String) { changeReason reason: String) {
// logger.debug("Connection changed candidate \(reason) \(remote.debugDescription) \(remote.description)") // logger.debug("Connection changed candidate \(reason) \(remote.debugDescription) \(remote.description)")
sendConnectedEvent(connection, local: local, remote: remote)
} }
func sendConnectedEvent(_ connection: WebRTC.RTCPeerConnection, local: WebRTC.RTCIceCandidate, remote: WebRTC.RTCIceCandidate) { func sendConnectedEvent(_ connection: WebRTC.RTCPeerConnection) {
connection.statistics { (stats: RTCStatisticsReport) in connection.statistics { (stats: RTCStatisticsReport) in
stats.statistics.values.forEach { stat in stats.statistics.values.forEach { stat in
// logger.debug("Stat \(stat.debugDescription)") // logger.debug("Stat \(stat.debugDescription)")
@ -517,24 +556,25 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
let localId = stat.values["localCandidateId"] as? String, let localId = stat.values["localCandidateId"] as? String,
let remoteId = stat.values["remoteCandidateId"] as? String, let remoteId = stat.values["remoteCandidateId"] as? String,
let localStats = stats.statistics[localId], let localStats = stats.statistics[localId],
let remoteStats = stats.statistics[remoteId], let remoteStats = stats.statistics[remoteId]
local.sdp.contains("\((localStats.values["ip"] as? String ?? "--")) \((localStats.values["port"] as? String ?? "--"))") &&
remote.sdp.contains("\((remoteStats.values["ip"] as? String ?? "--")) \((remoteStats.values["port"] as? String ?? "--"))")
{ {
Task { Task {
await self.sendCallResponse(.init( await self.sendCallResponse(.init(
corrId: nil, corrId: nil,
resp: .connected(connectionInfo: ConnectionInfo( resp: .connected(connectionInfo: ConnectionInfo(
localCandidate: local.toCandidate( localCandidate: RTCIceCandidate(
RTCIceCandidateType.init(rawValue: localStats.values["candidateType"] as! String), candidateType: RTCIceCandidateType.init(rawValue: localStats.values["candidateType"] as! String),
localStats.values["protocol"] as? String, protocol: localStats.values["protocol"] as? String,
localStats.values["relayProtocol"] as? String sdpMid: nil,
sdpMLineIndex: nil,
candidate: ""
), ),
remoteCandidate: remote.toCandidate( remoteCandidate: RTCIceCandidate(
RTCIceCandidateType.init(rawValue: remoteStats.values["candidateType"] as! String), candidateType: RTCIceCandidateType.init(rawValue: remoteStats.values["candidateType"] as! String),
remoteStats.values["protocol"] as? String, protocol: remoteStats.values["protocol"] as? String,
remoteStats.values["relayProtocol"] as? String sdpMid: nil,
))), sdpMLineIndex: nil,
candidate: ""))),
command: nil) command: nil)
) )
} }
@ -634,11 +674,10 @@ extension RTCIceCandidate {
} }
extension WebRTC.RTCIceCandidate { extension WebRTC.RTCIceCandidate {
func toCandidate(_ candidateType: RTCIceCandidateType?, _ protocol: String?, _ relayProtocol: String?) -> RTCIceCandidate { func toCandidate(_ candidateType: RTCIceCandidateType?, _ protocol: String?) -> RTCIceCandidate {
RTCIceCandidate( RTCIceCandidate(
candidateType: candidateType, candidateType: candidateType,
protocol: `protocol`, protocol: `protocol`,
relayProtocol: relayProtocol,
sdpMid: sdpMid, sdpMid: sdpMid,
sdpMLineIndex: Int(sdpMLineIndex), sdpMLineIndex: Int(sdpMLineIndex),
candidate: sdp candidate: sdp

View File

@ -370,7 +370,6 @@ fun CallInfoView(call: Call, alignment: Alignment.Horizontal) {
InfoText(call.callState.text) InfoText(call.callState.text)
val connInfo = call.connectionInfo val connInfo = call.connectionInfo
// val connInfoText = if (connInfo == null) "" else " (${connInfo.text}, ${connInfo.protocolText})"
val connInfoText = if (connInfo == null) "" else " (${connInfo.text})" val connInfoText = if (connInfo == null) "" else " (${connInfo.text})"
InfoText(call.encryptionStatus + connInfoText) InfoText(call.encryptionStatus + connInfoText)
} }
@ -585,8 +584,8 @@ fun PreviewActiveCallOverlayVideo() {
localMedia = CallMediaType.Video, localMedia = CallMediaType.Video,
peerMedia = CallMediaType.Video, peerMedia = CallMediaType.Video,
connectionInfo = ConnectionInfo( connectionInfo = ConnectionInfo(
RTCIceCandidate(RTCIceCandidateType.Host, "tcp", null), RTCIceCandidate(RTCIceCandidateType.Host, "tcp"),
RTCIceCandidate(RTCIceCandidateType.Host, "tcp", null) RTCIceCandidate(RTCIceCandidateType.Host, "tcp")
) )
), ),
speakerCanBeEnabled = true, speakerCanBeEnabled = true,
@ -611,8 +610,8 @@ fun PreviewActiveCallOverlayAudio() {
localMedia = CallMediaType.Audio, localMedia = CallMediaType.Audio,
peerMedia = CallMediaType.Audio, peerMedia = CallMediaType.Audio,
connectionInfo = ConnectionInfo( connectionInfo = ConnectionInfo(
RTCIceCandidate(RTCIceCandidateType.Host, "udp", null), RTCIceCandidate(RTCIceCandidateType.Host, "udp"),
RTCIceCandidate(RTCIceCandidateType.Host, "udp", null) RTCIceCandidate(RTCIceCandidateType.Host, "udp")
) )
), ),
speakerCanBeEnabled = true, speakerCanBeEnabled = true,

View File

@ -127,18 +127,10 @@ sealed class WCallResponse {
"${local?.value ?: "unknown"} / ${remote?.value ?: "unknown"}" "${local?.value ?: "unknown"} / ${remote?.value ?: "unknown"}"
} }
} }
val protocolText: String get() {
val local = localCandidate?.protocol?.uppercase(Locale.ROOT) ?: "unknown"
val localRelay = localCandidate?.relayProtocol?.uppercase(Locale.ROOT) ?: "unknown"
val remote = remoteCandidate?.protocol?.uppercase(Locale.ROOT) ?: "unknown"
val localText = if (localRelay == local || localCandidate?.relayProtocol == null) local else "$local ($localRelay)"
return if (local == remote) localText else "$localText / $remote"
}
} }
// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate // https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate
@Serializable data class RTCIceCandidate(val candidateType: RTCIceCandidateType?, val protocol: String?, val relayProtocol: String?) @Serializable data class RTCIceCandidate(val candidateType: RTCIceCandidateType?, val protocol: String?)
// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceServer // https://developer.mozilla.org/en-US/docs/Web/API/RTCIceServer
@Serializable data class RTCIceServer(val urls: List<String>, val username: String? = null, val credential: String? = null) @Serializable data class RTCIceServer(val urls: List<String>, val username: String? = null, val credential: String? = null)

View File

@ -136,7 +136,6 @@ private fun SendStateUpdates() {
.collect { call -> .collect { call ->
val state = call.callState.text val state = call.callState.text
val connInfo = call.connectionInfo val connInfo = call.connectionInfo
// val connInfoText = if (connInfo == null) "" else " (${connInfo.text}, ${connInfo.protocolText})"
val connInfoText = if (connInfo == null) "" else " (${connInfo.text})" val connInfoText = if (connInfo == null) "" else " (${connInfo.text})"
val description = call.encryptionStatus + connInfoText val description = call.encryptionStatus + connInfoText
chatModel.callCommand.add(WCallCommand.Description(state, description)) chatModel.callCommand.add(WCallCommand.Description(state, description))