From 97f5ba0145ea0f029e74916ef532bc647b9a5e42 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Fri, 17 May 2024 07:30:22 -0600 Subject: [PATCH] Update go2rtc and implement stream probe to only show 2 way talk when supported (#11407) * Support two way talk validation * Fix handling * Use go2rtc stream info to infer audio output * Update go2rtc * Update bundle policy * Formatting --- docker/main/Dockerfile | 2 +- docs/docs/configuration/advanced.md | 2 +- docs/docs/configuration/camera_specific.md | 2 +- docs/docs/configuration/restream.md | 4 +-- docs/docs/guides/configuring_go2rtc.md | 4 +-- docs/sidebars.js | 2 +- frigate/api/app.py | 17 +++++++++ web/src/components/player/WebRTCPlayer.tsx | 1 + web/src/types/live.ts | 27 ++++++++++++++ web/src/views/live/LiveCameraView.tsx | 42 +++++++++++++++++++--- 10 files changed, 91 insertions(+), 12 deletions(-) diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index de823bf1c..09a63e3b2 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -33,7 +33,7 @@ RUN --mount=type=tmpfs,target=/tmp --mount=type=tmpfs,target=/var/cache/apt \ FROM scratch AS go2rtc ARG TARGETARCH WORKDIR /rootfs/usr/local/go2rtc/bin -ADD --link --chmod=755 "https://github.com/AlexxIT/go2rtc/releases/download/v1.8.5/go2rtc_linux_${TARGETARCH}" go2rtc +ADD --link --chmod=755 "https://github.com/AlexxIT/go2rtc/releases/download/v1.9.2/go2rtc_linux_${TARGETARCH}" go2rtc #### diff --git a/docs/docs/configuration/advanced.md b/docs/docs/configuration/advanced.md index e0ed01948..51e14bd1d 100644 --- a/docs/docs/configuration/advanced.md +++ b/docs/docs/configuration/advanced.md @@ -120,7 +120,7 @@ NOTE: The folder that is mapped from the host needs to be the folder that contai ## Custom go2rtc version -Frigate currently includes go2rtc v1.8.5, there may be certain cases where you want to run a different version of go2rtc. +Frigate currently includes go2rtc v1.9.2, there may be certain cases where you want to run a different version of go2rtc. To do this: diff --git a/docs/docs/configuration/camera_specific.md b/docs/docs/configuration/camera_specific.md index f72ddcf7c..e884d2a73 100644 --- a/docs/docs/configuration/camera_specific.md +++ b/docs/docs/configuration/camera_specific.md @@ -175,7 +175,7 @@ go2rtc: - rtspx://192.168.1.1:7441/abcdefghijk ``` -[See the go2rtc docs for more information](https://github.com/AlexxIT/go2rtc/tree/v1.8.5#source-rtsp) +[See the go2rtc docs for more information](https://github.com/AlexxIT/go2rtc/tree/v1.9.2#source-rtsp) In the Unifi 2.0 update Unifi Protect Cameras had a change in audio sample rate which causes issues for ffmpeg. The input rate needs to be set for record if used directly with unifi protect. diff --git a/docs/docs/configuration/restream.md b/docs/docs/configuration/restream.md index 41c9cf39c..af6aaf9df 100644 --- a/docs/docs/configuration/restream.md +++ b/docs/docs/configuration/restream.md @@ -7,7 +7,7 @@ title: Restream Frigate can restream your video feed as an RTSP feed for other applications such as Home Assistant to utilize it at `rtsp://:8554/`. Port 8554 must be open. [This allows you to use a video feed for detection in Frigate and Home Assistant live view at the same time without having to make two separate connections to the camera](#reduce-connections-to-camera). The video feed is copied from the original video feed directly to avoid re-encoding. This feed does not include any annotation by Frigate. -Frigate uses [go2rtc](https://github.com/AlexxIT/go2rtc/tree/v1.8.5) to provide its restream and MSE/WebRTC capabilities. The go2rtc config is hosted at the `go2rtc` in the config, see [go2rtc docs](https://github.com/AlexxIT/go2rtc/tree/v1.8.5#configuration) for more advanced configurations and features. +Frigate uses [go2rtc](https://github.com/AlexxIT/go2rtc/tree/v1.9.2) to provide its restream and MSE/WebRTC capabilities. The go2rtc config is hosted at the `go2rtc` in the config, see [go2rtc docs](https://github.com/AlexxIT/go2rtc/tree/v1.9.2#configuration) for more advanced configurations and features. :::note @@ -134,7 +134,7 @@ cameras: ## Advanced Restream Configurations -The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.8.5#source-exec) source in go2rtc can be used for custom ffmpeg commands. An example is below: +The [exec](https://github.com/AlexxIT/go2rtc/tree/v1.9.2#source-exec) source in go2rtc can be used for custom ffmpeg commands. An example is below: NOTE: The output will need to be passed with two curly braces `{{output}}` diff --git a/docs/docs/guides/configuring_go2rtc.md b/docs/docs/guides/configuring_go2rtc.md index eef74d77e..5a0721f59 100644 --- a/docs/docs/guides/configuring_go2rtc.md +++ b/docs/docs/guides/configuring_go2rtc.md @@ -13,7 +13,7 @@ Use of the bundled go2rtc is optional. You can still configure FFmpeg to connect # Setup a go2rtc stream -First, you will want to configure go2rtc to connect to your camera stream by adding the stream you want to use for live view in your Frigate config file. If you set the stream name under go2rtc to match the name of your camera, it will automatically be mapped and you will get additional live view options for the camera. Avoid changing any other parts of your config at this step. Note that go2rtc supports [many different stream types](https://github.com/AlexxIT/go2rtc/tree/v1.8.5#module-streams), not just rtsp. +First, you will want to configure go2rtc to connect to your camera stream by adding the stream you want to use for live view in your Frigate config file. If you set the stream name under go2rtc to match the name of your camera, it will automatically be mapped and you will get additional live view options for the camera. Avoid changing any other parts of your config at this step. Note that go2rtc supports [many different stream types](https://github.com/AlexxIT/go2rtc/tree/v1.9.2#module-streams), not just rtsp. ```yaml go2rtc: @@ -26,7 +26,7 @@ The easiest live view to get working is MSE. After adding this to the config, re ### What if my video doesn't play? -If you are unable to see your video feed, first check the go2rtc logs in the Frigate UI under Logs in the sidebar. If go2rtc is having difficulty connecting to your camera, you should see some error messages in the log. If you do not see any errors, then the video codec of the stream may not be supported in your browser. If your camera stream is set to H265, try switching to H264. You can see more information about [video codec compatibility](https://github.com/AlexxIT/go2rtc/tree/v1.8.5#codecs-madness) in the go2rtc documentation. If you are not able to switch your camera settings from H265 to H264 or your stream is a different format such as MJPEG, you can use go2rtc to re-encode the video using the [FFmpeg parameters](https://github.com/AlexxIT/go2rtc/tree/v1.8.5#source-ffmpeg). It supports rotating and resizing video feeds and hardware acceleration. Keep in mind that transcoding video from one format to another is a resource intensive task and you may be better off using the built-in jsmpeg view. Here is an example of a config that will re-encode the stream to H264 without hardware acceleration: +If you are unable to see your video feed, first check the go2rtc logs in the Frigate UI under Logs in the sidebar. If go2rtc is having difficulty connecting to your camera, you should see some error messages in the log. If you do not see any errors, then the video codec of the stream may not be supported in your browser. If your camera stream is set to H265, try switching to H264. You can see more information about [video codec compatibility](https://github.com/AlexxIT/go2rtc/tree/v1.9.2#codecs-madness) in the go2rtc documentation. If you are not able to switch your camera settings from H265 to H264 or your stream is a different format such as MJPEG, you can use go2rtc to re-encode the video using the [FFmpeg parameters](https://github.com/AlexxIT/go2rtc/tree/v1.9.2#source-ffmpeg). It supports rotating and resizing video feeds and hardware acceleration. Keep in mind that transcoding video from one format to another is a resource intensive task and you may be better off using the built-in jsmpeg view. Here is an example of a config that will re-encode the stream to H264 without hardware acceleration: ```yaml go2rtc: diff --git a/docs/sidebars.js b/docs/sidebars.js index eff51212c..aceb6c207 100644 --- a/docs/sidebars.js +++ b/docs/sidebars.js @@ -23,7 +23,7 @@ module.exports = { { type: "link", label: "Go2RTC Configuration Reference", - href: "https://github.com/AlexxIT/go2rtc/tree/v1.8.5#configuration", + href: "https://github.com/AlexxIT/go2rtc/tree/v1.9.2#configuration", }, ], Detectors: [ diff --git a/frigate/api/app.py b/frigate/api/app.py index 67ad072a7..bd184d07d 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -117,6 +117,23 @@ def go2rtc_streams(): return jsonify(stream_data) +@bp.route("/go2rtc/streams/") +def go2rtc_camera_stream(camera_name: str): + r = requests.get( + f"http://127.0.0.1:1984/api/streams?src={camera_name}&video=all&audio=allµphone" + ) + if not r.ok: + logger.error("Failed to fetch streams from go2rtc") + return make_response( + jsonify({"success": False, "message": "Error fetching stream data"}), + 500, + ) + stream_data = r.json() + for producer in stream_data.get("producers", []): + producer["url"] = clean_camera_user_pass(producer.get("url", "")) + return jsonify(stream_data) + + @bp.route("/version") def version(): return VERSION diff --git a/web/src/components/player/WebRTCPlayer.tsx b/web/src/components/player/WebRTCPlayer.tsx index f98f9114f..8f966254b 100644 --- a/web/src/components/player/WebRTCPlayer.tsx +++ b/web/src/components/player/WebRTCPlayer.tsx @@ -40,6 +40,7 @@ export default function WebRtcPlayer({ } const pc = new RTCPeerConnection({ + bundlePolicy: "max-bundle", iceServers: [{ urls: "stun:stun.l.google.com:19302" }], }); diff --git a/web/src/types/live.ts b/web/src/types/live.ts index 34d6b17f6..451e4506c 100644 --- a/web/src/types/live.ts +++ b/web/src/types/live.ts @@ -3,3 +3,30 @@ export type VideoResolutionType = { width: number; height: number; }; + +type LiveProducerMetadata = { + type: string; + url: string; + remote_addr: string; + user_agent: string; + sdp: string; + medias: string[]; + receivers: string[]; + recv: number; +}; + +type LiveConsumerMetadata = { + type: string; + url: string; + remote_addr: string; + user_agent: string; + sdp: string; + medias: string[]; + senders: string[]; + send: number; +}; + +export type LiveStreamMetadata = { + producers: LiveProducerMetadata[]; + consumers: LiveConsumerMetadata[]; +}; diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index 2f6bb497f..9ec3800e2 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -21,7 +21,7 @@ import { TooltipProvider } from "@/components/ui/tooltip"; import { useResizeObserver } from "@/hooks/resize-observer"; import useKeyboardListener from "@/hooks/use-keyboard-listener"; import { CameraConfig } from "@/types/frigateConfig"; -import { VideoResolutionType } from "@/types/live"; +import { LiveStreamMetadata, VideoResolutionType } from "@/types/live"; import { CameraPtzInfo } from "@/types/ptz"; import { RecordingStartingPoint } from "@/types/record"; import React, { @@ -83,6 +83,40 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) { const [{ width: windowWidth, height: windowHeight }] = useResizeObserver(window); + // supported features + + const { data: cameraMetadata } = useSWR( + `go2rtc/streams/${camera.live.stream_name}`, + { + revalidateOnFocus: false, + }, + ); + + const supports2WayTalk = useMemo(() => { + if (!window.isSecureContext || !cameraMetadata) { + return false; + } + + return ( + cameraMetadata.producers.find( + (prod) => + prod.medias.find((media) => media.includes("audio, sendonly")) != undefined, + ) != undefined + ); + }, [cameraMetadata]); + const supportsAudioOutput = useMemo(() => { + if (!cameraMetadata) { + return false; + } + + return ( + cameraMetadata.producers.find( + (prod) => + prod.medias.find((media) => media.includes("audio, recvonly")) != undefined, + ) != undefined + ); + }, [cameraMetadata]) + // click overlay for ptzs const [clickOverlay, setClickOverlay] = useState(false); @@ -306,7 +340,7 @@ export default function LiveCameraView({ camera }: LiveCameraViewProps) { }} /> )} - {window.isSecureContext && ( + {supports2WayTalk && ( setMic(!mic)} /> )} - setAudio(!audio)} - /> + />}