IntenseFinance/simplex-chat
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

ios: fix calls connecting state (#3475)

Browse Source 
* ios: fix calls connecting state

* optimization

* changes

* removed relay protocol

* simplify

* use actor

* fix loop, better onChange, some questions

* remove extra iteration

---------

Co-authored-by: Avently <avently@local>
Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com>
This commit is contained in:
Stanislav Dmitrenko 2023-11-28 06:20:51 +08:00 committed by GitHub
parent 05278e5a06
commit 950bbe19da
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 102 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
apps/ios/Shared/Views/Call/ActiveCallView.swift
View File

@ -52,7 +52,12 @@ struct ActiveCallView: View {
AppDelegate.keepScreenOn(false)
client?.endCall()
}
.onChange(of: m.callCommand) { _ in sendCommandToClient()}
.onChange(of: m.callCommand) { cmd in
if let cmd = cmd {
m.callCommand = nil
sendCommandToClient(cmd)
}
}
.background(.black)
.preferredColorScheme(.dark)
}
@ -60,16 +65,17 @@ struct ActiveCallView: View {
private func createWebRTCClient() {
if client == nil && canConnectCall {
client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio)
sendCommandToClient()
if let cmd = m.callCommand {
m.callCommand = nil
sendCommandToClient(cmd)
}
}
}
private func sendCommandToClient() {
private func sendCommandToClient(_ cmd: WCallCommand) {
if call == m.activeCall,
m.activeCall != nil,
let client = client,
let cmd = m.callCommand {
m.callCommand = nil
let client = client {
logger.debug("sendCallCommand: \(cmd.cmdType)")
Task {
await client.sendCallCommand(command: cmd)
@ -255,7 +261,6 @@ struct ActiveCallOverlay: View {
HStack {
Text(call.encryptionStatus)
if let connInfo = call.connectionInfo {
// Text("(") + Text(connInfo.text) + Text(", \(connInfo.protocolText))")
Text("(") + Text(connInfo.text) + Text(")")
}
}

2
apps/ios/Shared/Views/Call/CallManager.swift
View File

@ -94,6 +94,8 @@ class CallManager {
completed()
} else {
logger.debug("CallManager.endCall: ending call...")
// TODO this command won't be executed because activeCall is assigned nil,
// and there is a condition in sendCommandToClient that would prevent its execution.
m.callCommand = .end
m.activeCall = nil
m.showCallView = false

14
apps/ios/Shared/Views/Call/WebRTC.swift
View File

@ -358,26 +358,12 @@ struct ConnectionInfo: Codable, Equatable {
return "\(local?.rawValue ?? unknown) / \(remote?.rawValue ?? unknown)"
}
}
var protocolText: String {
let unknown = NSLocalizedString("unknown", comment: "connection info")
let local = localCandidate?.protocol?.uppercased() ?? unknown
let localRelay = localCandidate?.relayProtocol?.uppercased() ?? unknown
let remote = remoteCandidate?.protocol?.uppercased() ?? unknown
let localText = localRelay == local || localCandidate?.relayProtocol == nil
? local
: "\(local) (\(localRelay))"
return local == remote
? localText
: "\(localText) / \(remote)"
}
}
// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate
struct RTCIceCandidate: Codable, Equatable {
var candidateType: RTCIceCandidateType?
var `protocol`: String?
var relayProtocol: String?
var sdpMid: String?
var sdpMLineIndex: Int?
var candidate: String

127
apps/ios/Shared/Views/Call/WebRTCClient.swift
View File

@ -21,7 +21,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
struct Call {
var connection: RTCPeerConnection
var iceCandidates: [RTCIceCandidate]
var iceCandidates: IceCandidates
var localMedia: CallMediaType
var localCamera: RTCVideoCapturer?
var localVideoSource: RTCVideoSource?
@ -33,6 +33,20 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
var frameDecryptor: RTCFrameDecryptor?
}
actor IceCandidates {
private var candidates: [RTCIceCandidate] = []
func getAndClear() async -> [RTCIceCandidate] {
let cs = candidates
candidates = []
return cs
}
func append(_ c: RTCIceCandidate) async {
candidates.append(c)
}
}
private let rtcAudioSession = RTCAudioSession.sharedInstance()
private let audioQueue = DispatchQueue(label: "audio")
private var sendCallResponse: (WVAPIMessage) async -> Void
@ -60,7 +74,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
WebRTC.RTCIceServer(urlStrings: ["turn:turn.simplex.im:443?transport=tcp"], username: "private", credential: "yleob6AVkiNI87hpR94Z"),
]
func initializeCall(_ iceServers: [WebRTC.RTCIceServer]?, _ remoteIceCandidates: [RTCIceCandidate], _ mediaType: CallMediaType, _ aesKey: String?, _ relay: Bool?) -> Call {
func initializeCall(_ iceServers: [WebRTC.RTCIceServer]?, _ mediaType: CallMediaType, _ aesKey: String?, _ relay: Bool?) -> Call {
let connection = createPeerConnection(iceServers ?? getWebRTCIceServers() ?? defaultIceServers, relay)
connection.delegate = self
createAudioSender(connection)
@ -87,7 +101,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
}
return Call(
connection: connection,
iceCandidates: remoteIceCandidates,
iceCandidates: IceCandidates(),
localMedia: mediaType,
localCamera: localCamera,
localVideoSource: localVideoSource,
@ -144,26 +158,21 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
logger.debug("starting incoming call - create webrtc session")
if activeCall.wrappedValue != nil { endCall() }
let encryption = WebRTCClient.enableEncryption
let call = initializeCall(iceServers?.toWebRTCIceServers(), [], media, encryption ? aesKey : nil, relay)
let call = initializeCall(iceServers?.toWebRTCIceServers(), media, encryption ? aesKey : nil, relay)
activeCall.wrappedValue = call
call.connection.offer { answer in
Task {
let gotCandidates = await self.waitWithTimeout(10_000, stepMs: 1000, until: { self.activeCall.wrappedValue?.iceCandidates.count ?? 0 > 0 })
if gotCandidates {
await self.sendCallResponse(.init(
corrId: nil,
resp: .offer(
offer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))),
iceCandidates: compressToBase64(input: encodeJSON(self.activeCall.wrappedValue?.iceCandidates ?? [])),
capabilities: CallCapabilities(encryption: encryption)
),
command: command)
)
} else {
self.endCall()
}
await self.sendCallResponse(.init(
corrId: nil,
resp: .offer(
offer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))),
iceCandidates: compressToBase64(input: encodeJSON(await self.getInitialIceCandidates())),
capabilities: CallCapabilities(encryption: encryption)
),
command: command)
)
await self.waitForMoreIceCandidates()
}
}
case let .offer(offer, iceCandidates, media, aesKey, iceServers, relay):
if activeCall.wrappedValue != nil {
@ -172,7 +181,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
resp = .error(message: "accept: encryption is not supported")
} else if let offer: CustomRTCSessionDescription = decodeJSON(decompressFromBase64(input: offer)),
let remoteIceCandidates: [RTCIceCandidate] = decodeJSON(decompressFromBase64(input: iceCandidates)) {
let call = initializeCall(iceServers?.toWebRTCIceServers(), remoteIceCandidates, media, WebRTCClient.enableEncryption ? aesKey : nil, relay)
let call = initializeCall(iceServers?.toWebRTCIceServers(), media, WebRTCClient.enableEncryption ? aesKey : nil, relay)
activeCall.wrappedValue = call
let pc = call.connection
if let type = offer.type, let sdp = offer.sdp {
@ -186,10 +195,11 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
corrId: nil,
resp: .answer(
answer: compressToBase64(input: encodeJSON(CustomRTCSessionDescription(type: answer.type.toSdpType(), sdp: answer.sdp))),
iceCandidates: compressToBase64(input: encodeJSON(call.iceCandidates))
iceCandidates: compressToBase64(input: encodeJSON(await self.getInitialIceCandidates()))
),
command: command)
)
await self.waitForMoreIceCandidates()
}
// }
}
@ -234,6 +244,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
resp = .ok
}
case .end:
// TODO possibly, endCall should be called before returning .ok
await sendCallResponse(.init(corrId: nil, resp: .ok, command: command))
endCall()
}
@ -242,6 +253,31 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
}
}
func getInitialIceCandidates() async -> [RTCIceCandidate] {
await untilIceComplete(timeoutMs: 750, stepMs: 150) {}
let candidates = await activeCall.wrappedValue?.iceCandidates.getAndClear() ?? []
logger.debug("WebRTCClient: sending initial ice candidates: \(candidates.count)")
return candidates
}
func waitForMoreIceCandidates() async {
await untilIceComplete(timeoutMs: 12000, stepMs: 1500) {
let candidates = await self.activeCall.wrappedValue?.iceCandidates.getAndClear() ?? []
if candidates.count > 0 {
logger.debug("WebRTCClient: sending more ice candidates: \(candidates.count)")
await self.sendIceCandidates(candidates)
}
}
}
func sendIceCandidates(_ candidates: [RTCIceCandidate]) async {
await self.sendCallResponse(.init(
corrId: nil,
resp: .ice(iceCandidates: compressToBase64(input: encodeJSON(candidates))),
command: nil)
)
}
func enableMedia(_ media: CallMediaType, _ enable: Bool) {
logger.debug("WebRTCClient: enabling media \(media.rawValue) \(enable)")
media == .video ? setVideoEnabled(enable) : setAudioEnabled(enable)
@ -387,12 +423,13 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg
audioSessionToDefaults()
}
func waitWithTimeout(_ timeoutMs: UInt64, stepMs: UInt64, until success: () -> Bool) async -> Bool {
let startedAt = DispatchTime.now()
while !success() && startedAt.uptimeNanoseconds + timeoutMs * 1000000 > DispatchTime.now().uptimeNanoseconds {
guard let _ = try? await Task.sleep(nanoseconds: stepMs * 1000000) else { break }
}
return success()
func untilIceComplete(timeoutMs: UInt64, stepMs: UInt64, action: @escaping () async -> Void) async {
var t: UInt64 = 0
repeat {
_ = try? await Task.sleep(nanoseconds: stepMs * 1000000)
t += stepMs
await action()
} while t < timeoutMs && activeCall.wrappedValue?.connection.iceGatheringState != .complete
}
}
@ -479,6 +516,7 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
default: enableSpeaker = false
}
setSpeakerEnabledAndConfigureSession(enableSpeaker)
case .connected: sendConnectedEvent(connection)
case .disconnected, .failed: endCall()
default: do {}
}
@ -491,7 +529,9 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
func peerConnection(_ connection: RTCPeerConnection, didGenerate candidate: WebRTC.RTCIceCandidate) {
// logger.debug("Connection generated candidate \(candidate.debugDescription)")
activeCall.wrappedValue?.iceCandidates.append(candidate.toCandidate(nil, nil, nil))
Task {
await self.activeCall.wrappedValue?.iceCandidates.append(candidate.toCandidate(nil, nil))
}
}
func peerConnection(_ connection: RTCPeerConnection, didRemove candidates: [WebRTC.RTCIceCandidate]) {
@ -506,10 +546,9 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
lastReceivedMs lastDataReceivedMs: Int32,
changeReason reason: String) {
// logger.debug("Connection changed candidate \(reason) \(remote.debugDescription) \(remote.description)")
sendConnectedEvent(connection, local: local, remote: remote)
}
func sendConnectedEvent(_ connection: WebRTC.RTCPeerConnection, local: WebRTC.RTCIceCandidate, remote: WebRTC.RTCIceCandidate) {
func sendConnectedEvent(_ connection: WebRTC.RTCPeerConnection) {
connection.statistics { (stats: RTCStatisticsReport) in
stats.statistics.values.forEach { stat in
// logger.debug("Stat \(stat.debugDescription)")
@ -517,24 +556,25 @@ extension WebRTCClient: RTCPeerConnectionDelegate {
let localId = stat.values["localCandidateId"] as? String,
let remoteId = stat.values["remoteCandidateId"] as? String,
let localStats = stats.statistics[localId],
let remoteStats = stats.statistics[remoteId],
local.sdp.contains("\((localStats.values["ip"] as? String ?? "--")) \((localStats.values["port"] as? String ?? "--"))") &&
remote.sdp.contains("\((remoteStats.values["ip"] as? String ?? "--")) \((remoteStats.values["port"] as? String ?? "--"))")
let remoteStats = stats.statistics[remoteId]
{
Task {
await self.sendCallResponse(.init(
corrId: nil,
resp: .connected(connectionInfo: ConnectionInfo(
localCandidate: local.toCandidate(
RTCIceCandidateType.init(rawValue: localStats.values["candidateType"] as! String),
localStats.values["protocol"] as? String,
localStats.values["relayProtocol"] as? String
localCandidate: RTCIceCandidate(
candidateType: RTCIceCandidateType.init(rawValue: localStats.values["candidateType"] as! String),
protocol: localStats.values["protocol"] as? String,
sdpMid: nil,
sdpMLineIndex: nil,
candidate: ""
),
remoteCandidate: remote.toCandidate(
RTCIceCandidateType.init(rawValue: remoteStats.values["candidateType"] as! String),
remoteStats.values["protocol"] as? String,
remoteStats.values["relayProtocol"] as? String
))),
remoteCandidate: RTCIceCandidate(
candidateType: RTCIceCandidateType.init(rawValue: remoteStats.values["candidateType"] as! String),
protocol: remoteStats.values["protocol"] as? String,
sdpMid: nil,
sdpMLineIndex: nil,
candidate: ""))),
command: nil)
)
}
@ -634,11 +674,10 @@ extension RTCIceCandidate {
}
extension WebRTC.RTCIceCandidate {
func toCandidate(_ candidateType: RTCIceCandidateType?, _ protocol: String?, _ relayProtocol: String?) -> RTCIceCandidate {
func toCandidate(_ candidateType: RTCIceCandidateType?, _ protocol: String?) -> RTCIceCandidate {
RTCIceCandidate(
candidateType: candidateType,
protocol: `protocol`,
relayProtocol: relayProtocol,
sdpMid: sdpMid,
sdpMLineIndex: Int(sdpMLineIndex),
candidate: sdp

9
apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt
View File

@ -370,7 +370,6 @@ fun CallInfoView(call: Call, alignment: Alignment.Horizontal) {
InfoText(call.callState.text)
val connInfo = call.connectionInfo
// val connInfoText = if (connInfo == null) "" else " (${connInfo.text}, ${connInfo.protocolText})"
val connInfoText = if (connInfo == null) "" else " (${connInfo.text})"
InfoText(call.encryptionStatus + connInfoText)
}
@ -585,8 +584,8 @@ fun PreviewActiveCallOverlayVideo() {
localMedia = CallMediaType.Video,
peerMedia = CallMediaType.Video,
connectionInfo = ConnectionInfo(
RTCIceCandidate(RTCIceCandidateType.Host, "tcp", null),
RTCIceCandidate(RTCIceCandidateType.Host, "tcp", null)
RTCIceCandidate(RTCIceCandidateType.Host, "tcp"),
RTCIceCandidate(RTCIceCandidateType.Host, "tcp")
)
),
speakerCanBeEnabled = true,
@ -611,8 +610,8 @@ fun PreviewActiveCallOverlayAudio() {
localMedia = CallMediaType.Audio,
peerMedia = CallMediaType.Audio,
connectionInfo = ConnectionInfo(
RTCIceCandidate(RTCIceCandidateType.Host, "udp", null),
RTCIceCandidate(RTCIceCandidateType.Host, "udp", null)
RTCIceCandidate(RTCIceCandidateType.Host, "udp"),
RTCIceCandidate(RTCIceCandidateType.Host, "udp")
)
),
speakerCanBeEnabled = true,

10
apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt
View File

@ -127,18 +127,10 @@ sealed class WCallResponse {
"${local?.value ?: "unknown"} / ${remote?.value ?: "unknown"}"
}
}
val protocolText: String get() {
val local = localCandidate?.protocol?.uppercase(Locale.ROOT) ?: "unknown"
val localRelay = localCandidate?.relayProtocol?.uppercase(Locale.ROOT) ?: "unknown"
val remote = remoteCandidate?.protocol?.uppercase(Locale.ROOT) ?: "unknown"
val localText = if (localRelay == local || localCandidate?.relayProtocol == null) local else "$local ($localRelay)"
return if (local == remote) localText else "$localText / $remote"
}
}
// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceCandidate
@Serializable data class RTCIceCandidate(val candidateType: RTCIceCandidateType?, val protocol: String?, val relayProtocol: String?)
@Serializable data class RTCIceCandidate(val candidateType: RTCIceCandidateType?, val protocol: String?)
// https://developer.mozilla.org/en-US/docs/Web/API/RTCIceServer
@Serializable data class RTCIceServer(val urls: List<String>, val username: String? = null, val credential: String? = null)

1
apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt
View File

@ -136,7 +136,6 @@ private fun SendStateUpdates() {
.collect { call ->
val state = call.callState.text
val connInfo = call.connectionInfo
// val connInfoText = if (connInfo == null) "" else " (${connInfo.text}, ${connInfo.protocolText})"
val connInfoText = if (connInfo == null) "" else " (${connInfo.text})"
val description = call.encryptionStatus + connInfoText
chatModel.callCommand.add(WCallCommand.Description(state, description))