diff --git a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt index 790345e97..8a5f6b9d0 100644 --- a/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt +++ b/apps/multiplatform/common/src/androidMain/kotlin/chat/simplex/common/views/call/CallView.android.kt @@ -132,6 +132,12 @@ actual fun ActiveCallView() { is WCallResponse.Ice -> withBGApi { chatModel.controller.apiSendCallExtraInfo(call.contact, r.iceCandidates) } + is WCallResponse.Media -> { + when (r.media) { + CallMediaType.Video -> call.remoteVideoEnabled.value = r.enable + CallMediaType.Audio -> call.remoteAudioEnabled.value = r.enable + } + } is WCallResponse.Connection -> try { val callStatus = json.decodeFromString("\"${r.state.connectionState}\"") @@ -275,8 +281,8 @@ private fun ActiveCallOverlayLayout( flipCamera: () -> Unit ) { Column(Modifier.padding(DEFAULT_PADDING)) { - when (call.peerMedia ?: call.localMedia) { - CallMediaType.Video -> { + when { + remember { call.remoteVideoEnabled }.value || (call.peerMedia ?: call.localMedia) == CallMediaType.Video -> { CallInfoView(call, alignment = Alignment.Start) Box(Modifier.fillMaxWidth().fillMaxHeight().weight(1f), contentAlignment = Alignment.BottomCenter) { DisabledBackgroundCallsButton() @@ -296,7 +302,7 @@ private fun ActiveCallOverlayLayout( } } } - CallMediaType.Audio -> { + else -> { Spacer(Modifier.fillMaxHeight().weight(1f)) Column( Modifier.fillMaxWidth(), diff --git a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt index 4be49d4c0..814e4f5ba 100644 --- a/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt +++ b/apps/multiplatform/common/src/commonMain/kotlin/chat/simplex/common/views/call/WebRTC.kt @@ -1,5 +1,7 @@ package chat.simplex.common.views.call +import androidx.compose.runtime.MutableState +import androidx.compose.runtime.mutableStateOf import chat.simplex.common.views.helpers.generalGetString import chat.simplex.common.model.* import chat.simplex.res.MR @@ -35,6 +37,9 @@ data class Call( } val hasMedia: Boolean get() = callState == CallState.OfferSent || callState == CallState.Negotiated || callState == CallState.Connected + + val remoteAudioEnabled: MutableState = mutableStateOf(true) + val remoteVideoEnabled: MutableState = mutableStateOf(localMedia == CallMediaType.Video) } enum class CallState { @@ -83,6 +88,7 @@ sealed class WCallResponse { @Serializable @SerialName("offer") data class Offer(val offer: String, val iceCandidates: String, val capabilities: CallCapabilities): WCallResponse() @Serializable @SerialName("answer") data class Answer(val answer: String, val iceCandidates: String): WCallResponse() @Serializable @SerialName("ice") data class Ice(val iceCandidates: String): WCallResponse() + @Serializable @SerialName("media") data class Media(val media: CallMediaType, val enable: Boolean): WCallResponse() @Serializable @SerialName("connection") data class Connection(val state: ConnectionState): WCallResponse() @Serializable @SerialName("connected") data class Connected(val connectionInfo: ConnectionInfo): WCallResponse() @Serializable @SerialName("end") object End: WCallResponse() diff --git a/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js b/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js index 833edf9e0..b13fda1b9 100644 --- a/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js +++ b/apps/multiplatform/common/src/commonMain/resources/assets/www/call.js @@ -396,11 +396,37 @@ const processCommand = (function () { console.log("set up decryption for receiving"); setupPeerTransform(TransformOperation.Decrypt, event.receiver, call.worker, call.aesKey, call.key); } + const hadAudio = call.remoteStream.getTracks().some((elem) => elem.kind == "audio" && elem.enabled); + const hadVideo = call.remoteStream.getTracks().some((elem) => elem.kind == "video" && elem.enabled); for (const stream of event.streams) { + stream.onaddtrack = (event) => { + console.log("LALAL ADDED TRACK " + event.track.kind); + }; for (const track of stream.getTracks()) { call.remoteStream.addTrack(track); } } + const hasAudio = call.remoteStream.getTracks().some((elem) => elem.kind == "audio" && elem.enabled); + const hasVideo = call.remoteStream.getTracks().some((elem) => elem.kind == "video" && elem.enabled); + console.log(`LALAL HAS AUDIO ${hasAudio} ${hasVideo} ${JSON.stringify(call.remoteStream.getTracks())}`); + if (hadAudio != hasAudio) { + const resp = { + type: "media", + media: CallMediaType.Audio, + enable: hasAudio, + }; + const apiResp = { corrId: undefined, resp, command: undefined }; + sendMessageToNative(apiResp); + } + if (hadVideo != hasVideo) { + const resp = { + type: "media", + media: CallMediaType.Video, + enable: hasVideo, + }; + const apiResp = { corrId: undefined, resp, command: undefined }; + sendMessageToNative(apiResp); + } console.log(`ontrack success`); } catch (e) { @@ -441,8 +467,6 @@ const processCommand = (function () { if (!videos) throw Error("no video elements"); const pc = call.connection; - const oldAudioTracks = call.localStream.getAudioTracks(); - const audioWasEnabled = oldAudioTracks.some((elem) => elem.enabled); let localStream; try { localStream = call.screenShareEnabled ? await getLocalScreenCaptureStream() : await getLocalMediaStream(call.localMedia, camera); @@ -458,24 +482,39 @@ const processCommand = (function () { call.localCamera = camera; const audioTracks = localStream.getAudioTracks(); const videoTracks = localStream.getVideoTracks(); - if (!audioWasEnabled && oldAudioTracks.length > 0) { - audioTracks.forEach((elem) => (elem.enabled = false)); + const audioWasEnabled = call.localStream.getAudioTracks().some((elem) => elem.enabled); + if (!audioWasEnabled && call.localStream.getAudioTracks().length > 0) { + enableMedia(localStream, CallMediaType.Audio, false); } if (!call.cameraEnabled && !call.screenShareEnabled) { - videoTracks.forEach((elem) => (elem.enabled = false)); + enableMedia(localStream, CallMediaType.Video, false); } - replaceTracks(pc, audioTracks); - replaceTracks(pc, videoTracks); + replaceTracks(pc, audioTracks, false); + replaceTracks(pc, videoTracks, call.screenShareEnabled); call.localStream = localStream; videos.local.srcObject = localStream; } - function replaceTracks(pc, tracks) { + function replaceTracks(pc, tracks, addIfNeeded) { + var _a; if (!tracks.length) return; const sender = pc.getSenders().find((s) => { var _a; return ((_a = s.track) === null || _a === void 0 ? void 0 : _a.kind) === tracks[0].kind; }); if (sender) for (const t of tracks) sender.replaceTrack(t); + else if (addIfNeeded) { + for (const track of tracks) + pc.addTrack(track, activeCall.localStream); + const call = activeCall; + if (call.aesKey && call.key) { + console.log("set up encryption for sending"); + for (const sender of pc.getSenders()) { + if (((_a = sender.track) === null || _a === void 0 ? void 0 : _a.kind) == "video") { + setupPeerTransform(TransformOperation.Encrypt, sender, call.worker, call.aesKey, call.key); + } + } + } + } } function setupPeerTransform(operation, peer, worker, aesKey, key) { if (worker && "RTCRtpScriptTransform" in window) { diff --git a/apps/multiplatform/common/src/commonMain/resources/assets/www/desktop/ui.js b/apps/multiplatform/common/src/commonMain/resources/assets/www/desktop/ui.js index 91bd71e9f..625f09293 100644 --- a/apps/multiplatform/common/src/commonMain/resources/assets/www/desktop/ui.js +++ b/apps/multiplatform/common/src/commonMain/resources/assets/www/desktop/ui.js @@ -74,9 +74,9 @@ function reactOnMessageFromServer(msg) { case "start": document.getElementById("toggle-audio").style.display = "inline-block"; document.getElementById("toggle-speaker").style.display = "inline-block"; + document.getElementById("toggle-screen").style.display = "inline-block"; if (msg.command.media == CallMediaType.Video) { document.getElementById("toggle-video").style.display = "inline-block"; - document.getElementById("toggle-screen").style.display = "inline-block"; } document.getElementById("info-block").className = msg.command.media; break; diff --git a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt index 42def0c75..4ad16277e 100644 --- a/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt +++ b/apps/multiplatform/common/src/desktopMain/kotlin/chat/simplex/common/views/call/CallView.desktop.kt @@ -67,6 +67,12 @@ actual fun ActiveCallView() { is WCallResponse.Ice -> withBGApi { chatModel.controller.apiSendCallExtraInfo(call.contact, r.iceCandidates) } + is WCallResponse.Media -> { + when (r.media) { + CallMediaType.Video -> call.remoteVideoEnabled.value = r.enable + CallMediaType.Audio -> call.remoteAudioEnabled.value = r.enable + } + } is WCallResponse.Connection -> try { val callStatus = json.decodeFromString("\"${r.state.connectionState}\"") diff --git a/packages/simplex-chat-webrtc/src/call.ts b/packages/simplex-chat-webrtc/src/call.ts index 1d67f685a..b13651ef9 100644 --- a/packages/simplex-chat-webrtc/src/call.ts +++ b/packages/simplex-chat-webrtc/src/call.ts @@ -23,6 +23,7 @@ type WCallResponse = | WCallOffer | WCallAnswer | WCallIceCandidates + | WCEnableMedia | WRConnection | WRCallConnected | WRCallEnd @@ -33,7 +34,18 @@ type WCallResponse = type WCallCommandTag = "capabilities" | "start" | "offer" | "answer" | "ice" | "media" | "camera" | "description" | "end" -type WCallResponseTag = "capabilities" | "offer" | "answer" | "ice" | "connection" | "connected" | "end" | "ended" | "ok" | "error" +type WCallResponseTag = + | "capabilities" + | "offer" + | "answer" + | "ice" + | "media" + | "connection" + | "connected" + | "end" + | "ended" + | "ok" + | "error" enum CallMediaType { Audio = "audio", @@ -98,7 +110,7 @@ interface WCallIceCandidates extends IWCallCommand, IWCallResponse { iceCandidates: string // JSON strings for RTCIceCandidateInit[] } -interface WCEnableMedia extends IWCallCommand { +interface WCEnableMedia extends IWCallCommand, IWCallResponse { type: "media" media: CallMediaType enable: boolean @@ -594,11 +606,37 @@ const processCommand = (function () { console.log("set up decryption for receiving") setupPeerTransform(TransformOperation.Decrypt, event.receiver as RTCRtpReceiverWithEncryption, call.worker, call.aesKey, call.key) } + const hadAudio = call.remoteStream.getTracks().some((elem) => elem.kind == "audio" && elem.enabled) + const hadVideo = call.remoteStream.getTracks().some((elem) => elem.kind == "video" && elem.enabled) for (const stream of event.streams) { + stream.onaddtrack = (event) => { + console.log("LALAL ADDED TRACK " + event.track.kind) + } for (const track of stream.getTracks()) { call.remoteStream.addTrack(track) } } + const hasAudio = call.remoteStream.getTracks().some((elem) => elem.kind == "audio" && elem.enabled) + const hasVideo = call.remoteStream.getTracks().some((elem) => elem.kind == "video" && elem.enabled) + console.log(`LALAL HAS AUDIO ${hasAudio} ${hasVideo} ${JSON.stringify(call.remoteStream.getTracks())}`) + if (hadAudio != hasAudio) { + const resp: WCEnableMedia = { + type: "media", + media: CallMediaType.Audio, + enable: hasAudio, + } + const apiResp: WVApiMessage = {corrId: undefined, resp, command: undefined} + sendMessageToNative(apiResp) + } + if (hadVideo != hasVideo) { + const resp: WCEnableMedia = { + type: "media", + media: CallMediaType.Video, + enable: hasVideo, + } + const apiResp: WVApiMessage = {corrId: undefined, resp, command: undefined} + sendMessageToNative(apiResp) + } console.log(`ontrack success`) } catch (e) { console.log(`ontrack error: ${(e as Error).message}`) @@ -639,8 +677,6 @@ const processCommand = (function () { const videos = getVideoElements() if (!videos) throw Error("no video elements") const pc = call.connection - const oldAudioTracks = call.localStream.getAudioTracks() - const audioWasEnabled = oldAudioTracks.some((elem) => elem.enabled) let localStream: MediaStream try { localStream = call.screenShareEnabled ? await getLocalScreenCaptureStream() : await getLocalMediaStream(call.localMedia, camera) @@ -655,23 +691,36 @@ const processCommand = (function () { const audioTracks = localStream.getAudioTracks() const videoTracks = localStream.getVideoTracks() - if (!audioWasEnabled && oldAudioTracks.length > 0) { - audioTracks.forEach((elem) => (elem.enabled = false)) + const audioWasEnabled = call.localStream.getAudioTracks().some((elem) => elem.enabled) + if (!audioWasEnabled && call.localStream.getAudioTracks().length > 0) { + enableMedia(localStream, CallMediaType.Audio, false) } if (!call.cameraEnabled && !call.screenShareEnabled) { - videoTracks.forEach((elem) => (elem.enabled = false)) + enableMedia(localStream, CallMediaType.Video, false) } - replaceTracks(pc, audioTracks) - replaceTracks(pc, videoTracks) + replaceTracks(pc, audioTracks, false) + replaceTracks(pc, videoTracks, call.screenShareEnabled) call.localStream = localStream videos.local.srcObject = localStream } - function replaceTracks(pc: RTCPeerConnection, tracks: MediaStreamTrack[]) { + function replaceTracks(pc: RTCPeerConnection, tracks: MediaStreamTrack[], addIfNeeded: boolean) { if (!tracks.length) return const sender = pc.getSenders().find((s) => s.track?.kind === tracks[0].kind) if (sender) for (const t of tracks) sender.replaceTrack(t) + else if (addIfNeeded) { + for (const track of tracks) pc.addTrack(track, activeCall!.localStream) + const call = activeCall! + if (call.aesKey && call.key) { + console.log("set up encryption for sending") + for (const sender of pc.getSenders() as RTCRtpSenderWithEncryption[]) { + if (sender.track?.kind == "video") { + setupPeerTransform(TransformOperation.Encrypt, sender, call.worker, call.aesKey, call.key) + } + } + } + } } function setupPeerTransform( diff --git a/packages/simplex-chat-webrtc/src/desktop/ui.ts b/packages/simplex-chat-webrtc/src/desktop/ui.ts index 1cabe7329..3d52c87d9 100644 --- a/packages/simplex-chat-webrtc/src/desktop/ui.ts +++ b/packages/simplex-chat-webrtc/src/desktop/ui.ts @@ -81,9 +81,9 @@ function reactOnMessageFromServer(msg: WVApiMessage) { case "start": document.getElementById("toggle-audio")!!.style.display = "inline-block" document.getElementById("toggle-speaker")!!.style.display = "inline-block" + document.getElementById("toggle-screen")!!.style.display = "inline-block" if (msg.command.media == CallMediaType.Video) { document.getElementById("toggle-video")!!.style.display = "inline-block" - document.getElementById("toggle-screen")!!.style.display = "inline-block" } document.getElementById("info-block")!!.className = msg.command.media break