From 0404b020e6fb36e114c0e86b17a3d09d0e37e5bc Mon Sep 17 00:00:00 2001 From: Stanislav Dmitrenko <7953703+avently@users.noreply.github.com> Date: Wed, 15 Mar 2023 13:21:21 +0300 Subject: [PATCH] ios: CallKit integrated with app lock and screen protect (#2007) * ios: CallKit integrated with app lock and screen protect * better lock mechanics * background color * logs * refactor, revert auth changes * additional state variable to allow connecting call * fix lock screen, public logs * show callkit option without dev tools --------- Co-authored-by: Evgeny Poberezkin <2769109+epoberezkin@users.noreply.github.com> --- apps/ios/Shared/ContentView.swift | 59 +++++++++++-------- apps/ios/Shared/SimpleXApp.swift | 9 ++- .../Shared/Views/Call/ActiveCallView.swift | 25 ++++++-- .../Shared/Views/Call/CallController.swift | 28 +++++++-- apps/ios/Shared/Views/Call/WebRTCClient.swift | 7 +++ .../Views/UserSettings/CallSettings.swift | 16 ++--- 6 files changed, 102 insertions(+), 42 deletions(-) diff --git a/apps/ios/Shared/ContentView.swift b/apps/ios/Shared/ContentView.swift index 2baf365c2..88b275872 100644 --- a/apps/ios/Shared/ContentView.swift +++ b/apps/ios/Shared/ContentView.swift @@ -13,8 +13,10 @@ struct ContentView: View { @EnvironmentObject var chatModel: ChatModel @ObservedObject var alertManager = AlertManager.shared @ObservedObject var callController = CallController.shared + @Environment(\.colorScheme) var colorScheme @Binding var doAuthenticate: Bool @Binding var userAuthorized: Bool? + @Binding var canConnectCall: Bool @AppStorage(DEFAULT_SHOW_LA_NOTICE) private var prefShowLANotice = false @AppStorage(DEFAULT_LA_NOTICE_SHOWN) private var prefLANoticeShown = false @AppStorage(DEFAULT_PERFORM_LA) private var prefPerformLA = false @@ -24,22 +26,29 @@ struct ContentView: View { var body: some View { ZStack { + if chatModel.showCallView, let call = chatModel.activeCall { + ActiveCallView(call: call, userAuthorized: $userAuthorized, canConnectCall: $canConnectCall) + } if prefPerformLA && userAuthorized != true { + Rectangle().fill(colorScheme == .dark ? .black : .white) + .frame(maxWidth: .infinity, maxHeight: .infinity) + .onTapGesture(perform: {}) Button(action: runAuthenticate) { Label("Unlock", systemImage: "lock") } } else if let status = chatModel.chatDbStatus, status != .ok { DatabaseErrorView(status: status) } else if !chatModel.v3DBMigration.startChat { MigrateToAppGroupView() - } else if let step = chatModel.onboardingStage { + } else if let step = chatModel.onboardingStage, (!chatModel.showCallView || chatModel.activeCall == nil) { if case .onboardingComplete = step, chatModel.currentUser != nil { - mainView().privacySensitive(protectScreen) + mainView() } else { OnboardingView(onboarding: step) } } } .onAppear { + logger.debug("ContentView: canConnectCall \(canConnectCall), doAuthenticate \(doAuthenticate)") if doAuthenticate { runAuthenticate() } } .onChange(of: doAuthenticate) { _ in if doAuthenticate { runAuthenticate() } } @@ -48,7 +57,7 @@ struct ContentView: View { private func mainView() -> some View { ZStack(alignment: .top) { - ChatListView() + ChatListView().privacySensitive(protectScreen) .onAppear { NtfManager.shared.requestAuthorization( onDeny: { @@ -75,31 +84,31 @@ struct ContentView: View { .sheet(isPresented: $showWhatsNew) { WhatsNewView() } - if chatModel.showCallView, let call = chatModel.activeCall { - ActiveCallView(call: call) - } IncomingCallView() } - .onContinueUserActivity("INStartCallIntent", perform: processUserActivity) - .onContinueUserActivity("INStartAudioCallIntent", perform: processUserActivity) - .onContinueUserActivity("INStartVideoCallIntent", perform: processUserActivity) +// .onContinueUserActivity("INStartCallIntent", perform: processUserActivity) +// .onContinueUserActivity("INStartAudioCallIntent", perform: processUserActivity) +// .onContinueUserActivity("INStartVideoCallIntent", perform: processUserActivity) } - private func processUserActivity(_ activity: NSUserActivity) { - let callToContact = { (contactId: ChatId?, mediaType: CallMediaType) in - if let chatInfo = chatModel.chats.first(where: { $0.id == contactId })?.chatInfo, - case let .direct(contact) = chatInfo { - CallController.shared.startCall(contact, mediaType) - } - } - if let intent = activity.interaction?.intent as? INStartCallIntent { - callToContact(intent.contacts?.first?.personHandle?.value, .audio) - } else if let intent = activity.interaction?.intent as? INStartAudioCallIntent { - callToContact(intent.contacts?.first?.personHandle?.value, .audio) - } else if let intent = activity.interaction?.intent as? INStartVideoCallIntent { - callToContact(intent.contacts?.first?.personHandle?.value, .video) - } - } +// private func processUserActivity(_ activity: NSUserActivity) { +// let intent = activity.interaction?.intent +// if let contacts = (intent as? INStartCallIntent)?.contacts { +// callToContact(contacts, .audio) +// } else if let contacts = (intent as? INStartAudioCallIntent)?.contacts { +// callToContact(contacts, .audio) +// } else if let contacts = (intent as? INStartVideoCallIntent)?.contacts { +// callToContact(contacts, .video) +// } +// } +// +// private func callToContact(_ contacts: [INPerson], _ mediaType: CallMediaType) { +// if let contactId = contacts.first?.personHandle?.value, +// let chatInfo = chatModel.chats.first(where: { $0.id == contactId })?.chatInfo, +// case let .direct(contact) = chatInfo { +// CallController.shared.startCall(contact, mediaType) +// } +// } private func runAuthenticate() { if !prefPerformLA { @@ -118,10 +127,12 @@ struct ContentView: View { switch (laResult) { case .success: userAuthorized = true + canConnectCall = true case .failed: break case .unavailable: userAuthorized = true + canConnectCall = true prefPerformLA = false AlertManager.shared.showAlert(laUnavailableTurningOffAlert()) } diff --git a/apps/ios/Shared/SimpleXApp.swift b/apps/ios/Shared/SimpleXApp.swift index a05da1ddf..ec216b806 100644 --- a/apps/ios/Shared/SimpleXApp.swift +++ b/apps/ios/Shared/SimpleXApp.swift @@ -20,6 +20,7 @@ struct SimpleXApp: App { @AppStorage(DEFAULT_PERFORM_LA) private var prefPerformLA = false @State private var userAuthorized: Bool? @State private var doAuthenticate = false + @State private var canConnectCall = false @State private var enteredBackground: TimeInterval? = nil init() { @@ -34,7 +35,7 @@ struct SimpleXApp: App { var body: some Scene { return WindowGroup { - ContentView(doAuthenticate: $doAuthenticate, userAuthorized: $userAuthorized) + ContentView(doAuthenticate: $doAuthenticate, userAuthorized: $userAuthorized, canConnectCall: $canConnectCall) .environmentObject(chatModel) .onOpenURL { url in logger.debug("ContentView.onOpenURL: \(url)") @@ -60,6 +61,7 @@ struct SimpleXApp: App { enteredBackground = ProcessInfo.processInfo.systemUptime } doAuthenticate = false + canConnectCall = false NtfManager.shared.setNtfBadgeCount(chatModel.totalUnreadCountForAllUsers()) case .active: CallController.shared.onEndCall = nil @@ -67,9 +69,12 @@ struct SimpleXApp: App { startChatAndActivate() if appState.inactive && chatModel.chatRunning == true { updateChats() - updateCallInvitations() + if !chatModel.showCallView && !CallController.shared.hasActiveCalls() { + updateCallInvitations() + } } doAuthenticate = authenticationExpired() + canConnectCall = !(doAuthenticate && prefPerformLA) default: break } diff --git a/apps/ios/Shared/Views/Call/ActiveCallView.swift b/apps/ios/Shared/Views/Call/ActiveCallView.swift index 1604ab9ad..9c8b256e0 100644 --- a/apps/ios/Shared/Views/Call/ActiveCallView.swift +++ b/apps/ios/Shared/Views/Call/ActiveCallView.swift @@ -12,7 +12,10 @@ import SimpleXChat struct ActiveCallView: View { @EnvironmentObject var m: ChatModel + @Environment(\.scenePhase) var scenePhase @ObservedObject var call: Call + @Binding var userAuthorized: Bool? + @Binding var canConnectCall: Bool @State private var client: WebRTCClient? = nil @State private var activeCall: WebRTCClient.Call? = nil @State private var localRendererAspectRatio: CGFloat? = nil @@ -36,12 +39,19 @@ struct ActiveCallView: View { } } .onAppear { - if client == nil { - client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio) - sendCommandToClient() - } + logger.debug("ActiveCallView: appear client is nil \(client == nil), userAuthorized \(userAuthorized.debugDescription, privacy: .public), scenePhase \(String(describing: scenePhase), privacy: .public)") + createWebRTCClient() + } + .onChange(of: userAuthorized) { _ in + logger.debug("ActiveCallView: userAuthorized changed to \(userAuthorized.debugDescription, privacy: .public)") + createWebRTCClient() + } + .onChange(of: canConnectCall) { _ in + logger.debug("ActiveCallView: canConnectCall changed to \(canConnectCall, privacy: .public)") + createWebRTCClient() } .onDisappear { + logger.debug("ActiveCallView: disappear") client?.endCall() } .onChange(of: m.callCommand) { _ in sendCommandToClient()} @@ -49,6 +59,13 @@ struct ActiveCallView: View { .preferredColorScheme(.dark) } + private func createWebRTCClient() { + if client == nil && ((userAuthorized == true && canConnectCall) || scenePhase == .background) { + client = WebRTCClient($activeCall, { msg in await MainActor.run { processRtcMessage(msg: msg) } }, $localRendererAspectRatio) + sendCommandToClient() + } + } + private func sendCommandToClient() { if call == m.activeCall, m.activeCall != nil, diff --git a/apps/ios/Shared/Views/Call/CallController.swift b/apps/ios/Shared/Views/Call/CallController.swift index cfeefdb90..a2fbe90c4 100644 --- a/apps/ios/Shared/Views/Call/CallController.swift +++ b/apps/ios/Shared/Views/Call/CallController.swift @@ -21,9 +21,9 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse private let provider = CXProvider(configuration: { let configuration = CXProviderConfiguration() - configuration.supportsVideo = false + configuration.supportsVideo = true configuration.supportedHandleTypes = [.generic] - configuration.includesCallsInRecents = UserDefaults.standard.bool(forKey: DEFAULT_CALL_KIT_CALLS_IN_RECENTS) + configuration.includesCallsInRecents = false // UserDefaults.standard.bool(forKey: DEFAULT_CALL_KIT_CALLS_IN_RECENTS) configuration.maximumCallGroups = 1 configuration.maximumCallsPerCallGroup = 1 configuration.iconTemplateImageData = UIImage(named: "icon-transparent")?.pngData() @@ -98,6 +98,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse func provider(_ provider: CXProvider, didActivate audioSession: AVAudioSession) { print("received", #function) + logger.debug("CallController: activating audioSession and audio in WebRTCClient") RTCAudioSession.sharedInstance().audioSessionDidActivate(audioSession) RTCAudioSession.sharedInstance().isAudioEnabled = true do { @@ -113,10 +114,12 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse func provider(_ provider: CXProvider, didDeactivate audioSession: AVAudioSession) { print("received", #function) + logger.debug("CallController: deactivating audioSession and audio in WebRTCClient") RTCAudioSession.sharedInstance().audioSessionDidDeactivate(audioSession) RTCAudioSession.sharedInstance().isAudioEnabled = false do { try audioSession.setActive(false) + logger.debug("audioSession deactivated") } catch { print(error) logger.error("failed deactivating audio session") @@ -125,6 +128,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse // see `.onChange(of: scenePhase)` in SimpleXApp DispatchQueue.main.asyncAfter(deadline: .now() + 3) { [weak self] in if ChatModel.shared.activeCall == nil { + logger.debug("CallController: calling callback onEndCall which is \(self?.onEndCall == nil ? "nil" : "non-nil", privacy: .public)") self?.onEndCall?() } } @@ -136,14 +140,17 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func pushRegistry(_ registry: PKPushRegistry, didReceiveIncomingPushWith payload: PKPushPayload, for type: PKPushType, completion: @escaping () -> Void) { + logger.debug("CallController: did receive push with type \(type.rawValue, privacy: .public)") if type == .voIP { if (!ChatModel.shared.chatInitialized) { + logger.debug("CallController: initializing chat and returning") initChatAndMigrate() startChatAndActivate() CallController.shared.onEndCall = { terminateChat() } // CallKit will be called from different place, see SimpleXAPI.startChat() return } else { + logger.debug("CallController: starting chat (already initialized)") startChatAndActivate() CallController.shared.onEndCall = { suspendChat() @@ -162,6 +169,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse callUpdate.remoteHandle = CXHandle(type: .generic, value: contactId) callUpdate.localizedCallerName = displayName callUpdate.hasVideo = media == CallMediaType.video.rawValue + logger.debug("CallController: reporting incoming call directly to CallKit") CallController.shared.provider.reportNewIncomingCall(with: uuid, update: callUpdate, completion: { error in if error != nil { ChatModel.shared.callInvitations.removeValue(forKey: contactId) @@ -174,7 +182,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func reportNewIncomingCall(invitation: RcvCallInvitation, completion: @escaping (Error?) -> Void) { - logger.debug("CallController.reportNewIncomingCall, UUID=\(String(describing: invitation.callkitUUID))") + logger.debug("CallController.reportNewIncomingCall, UUID=\(String(describing: invitation.callkitUUID), privacy: .public)") if CallController.useCallKit(), let uuid = invitation.callkitUUID { let update = CXCallUpdate() update.remoteHandle = CXHandle(type: .generic, value: invitation.contact.id) @@ -190,6 +198,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func reportIncomingCall(call: Call, connectedAt dateConnected: Date?) { + logger.debug("CallController: reporting incoming call connected") if CallController.useCallKit() { // Fulfilling this action only after connect, otherwise there are no audio and mic on lockscreen fulfillOnConnect?.fulfill() @@ -198,12 +207,14 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func reportOutgoingCall(call: Call, connectedAt dateConnected: Date?) { + logger.debug("CallController: reporting outgoing call connected") if CallController.useCallKit(), let uuid = call.callkitUUID { provider.reportOutgoingCall(with: uuid, connectedAt: dateConnected) } } func reportCallRemoteEnded(invitation: RcvCallInvitation) { + logger.debug("CallController: reporting remote ended") if CallController.useCallKit(), let uuid = invitation.callkitUUID { provider.reportCall(with: uuid, endedAt: nil, reason: .remoteEnded) } else if invitation.contact.id == activeCallInvitation?.contact.id { @@ -212,6 +223,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func reportCallRemoteEnded(call: Call) { + logger.debug("CallController: reporting remote ended") if CallController.useCallKit(), let uuid = call.callkitUUID { provider.reportCall(with: uuid, endedAt: nil, reason: .remoteEnded) } @@ -241,6 +253,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func answerCall(invitation: RcvCallInvitation) { + logger.debug("CallController: answering a call") if CallController.useCallKit(), let callUUID = invitation.callkitUUID { requestTransaction(with: CXAnswerCallAction(call: callUUID)) } else { @@ -252,6 +265,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func endCall(callUUID: UUID) { + logger.debug("CallController: ending the call") if CallController.useCallKit() { requestTransaction(with: CXEndCallAction(call: callUUID)) } else { @@ -266,6 +280,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func endCall(invitation: RcvCallInvitation) { + logger.debug("CallController: ending the call") callManager.endCall(invitation: invitation) { if invitation.contact.id == self.activeCallInvitation?.contact.id { DispatchQueue.main.async { @@ -276,6 +291,7 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse } func endCall(call: Call, completed: @escaping () -> Void) { + logger.debug("CallController: ending the call") callManager.endCall(call: call, completed: completed) } @@ -292,10 +308,14 @@ class CallController: NSObject, CXProviderDelegate, PKPushRegistryDelegate, Obse provider.configuration = conf } + func hasActiveCalls() -> Bool { + controller.callObserver.calls.count > 0 + } + private func requestTransaction(with action: CXAction, onSuccess: @escaping () -> Void = {}) { controller.request(CXTransaction(action: action)) { error in if let error = error { - logger.error("CallController.requestTransaction error requesting transaction: \(error.localizedDescription)") + logger.error("CallController.requestTransaction error requesting transaction: \(error.localizedDescription, privacy: .public)") } else { logger.debug("CallController.requestTransaction requested transaction successfully") onSuccess() diff --git a/apps/ios/Shared/Views/Call/WebRTCClient.swift b/apps/ios/Shared/Views/Call/WebRTCClient.swift index 7118d04a7..f64276f9b 100644 --- a/apps/ios/Shared/Views/Call/WebRTCClient.swift +++ b/apps/ios/Shared/Views/Call/WebRTCClient.swift @@ -49,6 +49,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg self.localRendererAspectRatio = localRendererAspectRatio rtcAudioSession.useManualAudio = CallController.useCallKit() rtcAudioSession.isAudioEnabled = !CallController.useCallKit() + logger.debug("WebRTCClient: rtcAudioSession has manual audio \(self.rtcAudioSession.useManualAudio) and audio enabled \(self.rtcAudioSession.isAudioEnabled)}") super.init() } @@ -241,6 +242,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg } func enableMedia(_ media: CallMediaType, _ enable: Bool) { + logger.debug("WebRTCClient: enabling media \(media.rawValue) \(enable)") media == .video ? setVideoEnabled(enable) : setAudioEnabled(enable) } @@ -363,6 +365,7 @@ final class WebRTCClient: NSObject, RTCVideoViewDelegate, RTCFrameEncryptorDeleg func endCall() { guard let call = activeCall.wrappedValue else { return } + logger.debug("WebRTCClient: ending the call") activeCall.wrappedValue = nil call.connection.close() call.connection.delegate = nil @@ -534,6 +537,7 @@ extension WebRTCClient { } func setSpeakerEnabledAndConfigureSession( _ enabled: Bool) { + logger.debug("WebRTCClient: configuring session with speaker enabled \(enabled)") audioQueue.async { [weak self] in guard let self = self else { return } self.rtcAudioSession.lockForConfiguration() @@ -545,6 +549,7 @@ extension WebRTCClient { try self.rtcAudioSession.setMode(AVAudioSession.Mode.voiceChat.rawValue) try self.rtcAudioSession.overrideOutputAudioPort(enabled ? .speaker : .none) try self.rtcAudioSession.setActive(true) + logger.debug("WebRTCClient: configuring session with speaker enabled \(enabled) success") } catch let error { logger.debug("Error configuring AVAudioSession: \(error)") } @@ -552,6 +557,7 @@ extension WebRTCClient { } func audioSessionToDefaults() { + logger.debug("WebRTCClient: audioSession to defaults") audioQueue.async { [weak self] in guard let self = self else { return } self.rtcAudioSession.lockForConfiguration() @@ -563,6 +569,7 @@ extension WebRTCClient { try self.rtcAudioSession.setMode(AVAudioSession.Mode.default.rawValue) try self.rtcAudioSession.overrideOutputAudioPort(.none) try self.rtcAudioSession.setActive(false) + logger.debug("WebRTCClient: audioSession to defaults success") } catch let error { logger.debug("Error configuring AVAudioSession with defaults: \(error)") } diff --git a/apps/ios/Shared/Views/UserSettings/CallSettings.swift b/apps/ios/Shared/Views/UserSettings/CallSettings.swift index cfc18011b..9d3f56c71 100644 --- a/apps/ios/Shared/Views/UserSettings/CallSettings.swift +++ b/apps/ios/Shared/Views/UserSettings/CallSettings.swift @@ -22,16 +22,16 @@ struct CallSettings: View { Section { Toggle("Connect via relay", isOn: $webrtcPolicyRelay) - if !CallController.isInChina && developerTools { + if !CallController.isInChina { Toggle("Use CallKit", isOn: $callKitEnabled) - if allowChangingCallsHistory { - Toggle("Show calls in phone history", isOn: $callKitCallsInRecents) - .disabled(!callKitEnabled) - .onChange(of: callKitCallsInRecents) { value in - CallController.shared.showInRecents(value) - } - } +// if allowChangingCallsHistory { +// Toggle("Show calls in phone history", isOn: $callKitCallsInRecents) +// .disabled(!callKitEnabled) +// .onChange(of: callKitCallsInRecents) { value in +// CallController.shared.showInRecents(value) +// } +// } } NavigationLink {