Improve handling of backchannel audio in camera wizard (#21250)

* Improve handling of backchannel audio in camera wizard

* Cleanup

* look for backchannel on all registered streams on save

avoids potential issues with a timeout in stream registration

---------

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>

web/src/components/settings/CameraStreamingDialog.tsx

@@ -34,6 +34,7 @@ import { LiveStreamMetadata } from "@/types/live";
 import { Trans, useTranslation } from "react-i18next";
 import { useDocDomain } from "@/hooks/use-doc-domain";
 import { useCameraFriendlyName } from "@/hooks/use-camera-friendly-name";
+import { detectCameraAudioFeatures } from "@/utils/cameraUtil";
 
 type CameraStreamingDialogProps = {
   camera: string;
@@ -80,20 +81,10 @@ export function CameraStreamingDialog({
 
   const cameraMetadata = streamName ? streamMetadata?.[streamName] : undefined;
 
-  const supportsAudioOutput = useMemo(() => {
+  const { audioOutput: supportsAudioOutput } = useMemo(
-    if (!cameraMetadata) {
+    () => detectCameraAudioFeatures(cameraMetadata),
-      return false;
+    [cameraMetadata],
-    }
+  );
-
-    return (
-      cameraMetadata.producers.find(
-        (prod) =>
-          prod.medias &&
-          prod.medias.find((media) => media.includes("audio, recvonly")) !=
-            undefined,
-      ) != undefined
-    );
-  }, [cameraMetadata]);
 
   // handlers
 

web/src/components/settings/CameraWizardDialog.tsx

@@ -237,7 +237,18 @@ export default function CameraWizardDialog({
                 const streamUrl = stream.useFfmpeg
                   ? `ffmpeg:${stream.url}`
                   : stream.url;
-                go2rtcStreams[streamName] = [streamUrl];
+
+                if (wizardData.hasBackchannel ?? false) {
+                  // Add two streams: one with #backchannel=0 and one without
+                  // in order to avoid taking control of the microphone during connections
+                  go2rtcStreams[streamName] = [
+                    `${streamUrl}#backchannel=0`,
+                    streamUrl,
+                  ];
+                } else {
+                  // Add single stream as normal
+                  go2rtcStreams[streamName] = [streamUrl];
+                }
               });
 
               if (Object.keys(go2rtcStreams).length > 0) {

web/src/components/settings/wizard/Step4Validation.tsx

@@ -14,7 +14,8 @@ import axios from "axios";
 import { toast } from "sonner";
 import MSEPlayer from "@/components/player/MsePlayer";
 import { WizardFormData, StreamConfig, TestResult } from "@/types/cameraWizard";
-import { PlayerStatsType } from "@/types/live";
+import { PlayerStatsType, LiveStreamMetadata } from "@/types/live";
+import { detectCameraAudioFeatures } from "@/utils/cameraUtil";
 import { FaCircleCheck, FaTriangleExclamation } from "react-icons/fa6";
 import { LuX } from "react-icons/lu";
 import { Card, CardContent } from "../../ui/card";
@@ -41,6 +42,9 @@ export default function Step4Validation({
   const [measuredBandwidth, setMeasuredBandwidth] = useState<
     Map<string, number>
   >(new Map());
+  const [registeredStreamIds, setRegisteredStreamIds] = useState<
+    Map<string, string>
+  >(new Map());
 
   const streams = useMemo(() => wizardData.streams || [], [wizardData.streams]);
 
@@ -125,6 +129,27 @@ export default function Step4Validation({
     [],
   );
 
+  const checkBackchannel = useCallback(
+    async (go2rtcStreamId: string, useFfmpeg: boolean): Promise<boolean> => {
+      // ffmpeg compatibility mode guarantees no backchannel connection
+      if (useFfmpeg) {
+        return false;
+      }
+
+      try {
+        const response = await axios.get<LiveStreamMetadata>(
+          `go2rtc/streams/${go2rtcStreamId}`,
+        );
+
+        const audioFeatures = detectCameraAudioFeatures(response.data, false);
+        return audioFeatures.twoWayAudio;
+      } catch {
+        return false;
+      }
+    },
+    [],
+  );
+
   const validateStream = useCallback(
     async (stream: StreamConfig) => {
       if (!stream.url.trim()) {
@@ -208,12 +233,31 @@ export default function Step4Validation({
     }
   }, [streams, onUpdate, t, performStreamValidation]);
 
-  const handleSave = useCallback(() => {
+  const handleSave = useCallback(async () => {
     if (!wizardData.cameraName || !wizardData.streams?.length) {
       toast.error(t("cameraWizard.step4.saveError"));
       return;
     }
 
+    const candidateStreams =
+      wizardData.streams?.filter(
+        (s) => s.testResult?.success && !(s.useFfmpeg ?? false),
+      ) || [];
+
+    let hasBackchannelResult = false;
+    if (candidateStreams.length > 0) {
+      // Check all candidate streams for backchannel support
+      const backchanelChecks = candidateStreams.map((stream) => {
+        const actualStreamId = registeredStreamIds.get(stream.id);
+        return actualStreamId
+          ? checkBackchannel(actualStreamId, stream.useFfmpeg ?? false)
+          : Promise.resolve(false);
+      });
+      const results = await Promise.all(backchanelChecks);
+      hasBackchannelResult = results.some((result) => result);
+    }
+    onUpdate({ hasBackchannel: hasBackchannelResult });
+
     // Convert wizard data to final config format
     const configData = {
       cameraName: wizardData.cameraName,
@@ -223,10 +267,11 @@ export default function Step4Validation({
       brandTemplate: wizardData.brandTemplate,
       customUrl: wizardData.customUrl,
       streams: wizardData.streams,
+      hasBackchannel: wizardData.hasBackchannel,
     };
 
     onSave(configData);
-  }, [wizardData, onSave, t]);
+  }, [wizardData, onSave, t, onUpdate, checkBackchannel, registeredStreamIds]);
 
   const canSave = useMemo(() => {
     return (
@@ -324,6 +369,11 @@ export default function Step4Validation({
                       <StreamPreview
                         stream={stream}
                         onBandwidthUpdate={handleBandwidthUpdate}
+                        onStreamRegistered={(go2rtcStreamId) => {
+                          setRegisteredStreamIds((prev) =>
+                            new Map(prev).set(stream.id, go2rtcStreamId),
+                          );
+                        }}
                       />
                     </div>
                   )}
@@ -683,10 +733,15 @@ function BandwidthDisplay({
 type StreamPreviewProps = {
   stream: StreamConfig;
   onBandwidthUpdate?: (streamId: string, bandwidth: number) => void;
+  onStreamRegistered?: (go2rtcStreamId: string) => void;
 };
 
 // live stream preview using MSEPlayer with temp go2rtc streams
-function StreamPreview({ stream, onBandwidthUpdate }: StreamPreviewProps) {
+function StreamPreview({
+  stream,
+  onBandwidthUpdate,
+  onStreamRegistered,
+}: StreamPreviewProps) {
   const { t } = useTranslation(["views/settings"]);
   const [streamId, setStreamId] = useState(`wizard_${stream.id}_${Date.now()}`);
   const [registered, setRegistered] = useState(false);
@@ -722,10 +777,11 @@ function StreamPreview({ stream, onBandwidthUpdate }: StreamPreviewProps) {
       .put(`go2rtc/streams/${streamId}`, null, {
         params: { src: streamUrl },
       })
-      .then(() => {
+      .then(async () => {
         // Add small delay to allow go2rtc api to run and initialize the stream
         setTimeout(() => {
           setRegistered(true);
+          onStreamRegistered?.(streamId);
         }, 500);
       })
       .catch(() => {
@@ -738,6 +794,7 @@ function StreamPreview({ stream, onBandwidthUpdate }: StreamPreviewProps) {
         // do nothing on cleanup errors - go2rtc won't consume the streams
       });
     };
+    // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [stream.url, stream.useFfmpeg, streamId]);
 
   const resolution = stream.testResult?.resolution;

web/src/hooks/use-camera-live-mode.ts

@@ -3,6 +3,7 @@ import { useCallback, useEffect, useState, useMemo } from "react";
 import useSWR from "swr";
 import { LivePlayerMode } from "@/types/live";
 import useDeferredStreamMetadata from "./use-deferred-stream-metadata";
+import { detectCameraAudioFeatures } from "@/utils/cameraUtil";
 
 export default function useCameraLiveMode(
   cameras: CameraConfig[],
@@ -83,16 +84,9 @@ export default function useCameraLiveMode(
       if (isRestreamed) {
         Object.values(camera.live.streams).forEach((streamName) => {
           const metadata = streamMetadata[streamName];
+          const audioFeatures = detectCameraAudioFeatures(metadata);
           newSupportsAudioOutputStates[streamName] = {
-            supportsAudio: metadata
+            supportsAudio: audioFeatures.audioOutput,
-              ? metadata.producers.find(
-                  (prod) =>
-                    prod.medias &&
-                    prod.medias.find((media) =>
-                      media.includes("audio, recvonly"),
-                    ) !== undefined,
-                ) !== undefined
-              : false,
             cameraName: camera.name,
           };
         });

web/src/types/cameraWizard.ts

@@ -118,6 +118,7 @@ export type WizardFormData = {
   probeResult?: OnvifProbeResponse;
   probeCandidates?: string[]; // candidate URLs from probe
   candidateTests?: CandidateTestMap; // test results for candidates
+  hasBackchannel?: boolean; // true if camera supports backchannel audio
 };
 
 // API Response Types

web/src/utils/cameraUtil.ts

@@ -1,5 +1,6 @@
 import { baseUrl } from "@/api/baseUrl";
 import { generateFixedHash, isValidId } from "./stringUtil";
+import type { LiveStreamMetadata } from "@/types/live";
 
 /**
  * Processes a user-entered camera name and returns both the final camera name
@@ -27,8 +28,6 @@ export function processCameraName(userInput: string): {
   };
 }
 
-// ==================== Reolink Camera Detection ====================
-
 /**
  * Detect Reolink camera capabilities and recommend optimal protocol
  *
@@ -98,3 +97,53 @@ export function maskUri(uri: string): string {
   }
   return uri;
 }
+
+/**
+ * Represents the audio features supported by a camera stream
+ */
+export type CameraAudioFeatures = {
+  twoWayAudio: boolean;
+  audioOutput: boolean;
+};
+
+/**
+ * Detects camera audio features from go2rtc stream metadata.
+ * Checks for two-way audio (backchannel) and audio output capabilities.
+ *
+ * @param metadata - The LiveStreamMetadata from go2rtc stream
+ * @param requireSecureContext - If true, two-way audio requires secure context (default: true)
+ * @returns CameraAudioFeatures object with detected capabilities
+ */
+export function detectCameraAudioFeatures(
+  metadata: LiveStreamMetadata | null | undefined,
+  requireSecureContext: boolean = true,
+): CameraAudioFeatures {
+  if (!metadata) {
+    return {
+      twoWayAudio: false,
+      audioOutput: false,
+    };
+  }
+
+  const twoWayAudio =
+    (!requireSecureContext || window.isSecureContext) &&
+    metadata.producers.find(
+      (prod) =>
+        prod.medias &&
+        prod.medias.find((media) => media.includes("audio, sendonly")) !=
+          undefined,
+    ) != undefined;
+
+  const audioOutput =
+    metadata.producers.find(
+      (prod) =>
+        prod.medias &&
+        prod.medias.find((media) => media.includes("audio, recvonly")) !=
+          undefined,
+    ) != undefined;
+
+  return {
+    twoWayAudio: !!twoWayAudio,
+    audioOutput: !!audioOutput,
+  };
+}

web/src/views/live/LiveCameraView.tsx

@@ -110,6 +110,7 @@ import { Toaster } from "@/components/ui/sonner";
 import { useIsAdmin } from "@/hooks/use-is-admin";
 import { useTranslation } from "react-i18next";
 import { useDocDomain } from "@/hooks/use-doc-domain";
+import { detectCameraAudioFeatures } from "@/utils/cameraUtil";
 import PtzControlPanel from "@/components/overlay/PtzControlPanel";
 import ObjectSettingsView from "../settings/ObjectSettingsView";
 import { useSearchEffect } from "@/hooks/use-overlay-state";
@@ -168,34 +169,8 @@ export default function LiveCameraView({
     },
   );
 
-  const supports2WayTalk = useMemo(() => {
+  const { twoWayAudio: supports2WayTalk, audioOutput: supportsAudioOutput } =
-    if (!window.isSecureContext || !cameraMetadata) {
+    useMemo(() => detectCameraAudioFeatures(cameraMetadata), [cameraMetadata]);
-      return false;
-    }
-
-    return (
-      cameraMetadata.producers.find(
-        (prod) =>
-          prod.medias &&
-          prod.medias.find((media) => media.includes("audio, sendonly")) !=
-            undefined,
-      ) != undefined
-    );
-  }, [cameraMetadata]);
-  const supportsAudioOutput = useMemo(() => {
-    if (!cameraMetadata) {
-      return false;
-    }
-
-    return (
-      cameraMetadata.producers.find(
-        (prod) =>
-          prod.medias &&
-          prod.medias.find((media) => media.includes("audio, recvonly")) !=
-            undefined,
-      ) != undefined
-    );
-  }, [cameraMetadata]);
 
   // camera enabled state
   const { payload: enabledState } = useEnabledState(camera.name);