Live view improvements (#24967)

* hold the last frame on screen while switching live streams

Switching streams in the single camera view blanked the player until the new stream played, because the old player unmounts in the same render and the hidden new player gave the card no height. The outgoing video's last frame is now drawn to a canvas before the remount and covers the gap until the new stream plays, and the card takes the camera's aspect ratio so it keeps its size while the video is hidden.

* show live streams as a one-tap grid in the mobile drawer

Switching streams on mobile meant scrolling past the camera toggles, opening a dropdown, and picking a stream. The drawer now opens with the stream section, and streams are a wrapping grid of toggles sized to their names, so a switch is two taps. A single stream shows as plain text. The audio and two-way talk lines show a spinner while the stream's go2rtc metadata loads instead of reporting audio as unavailable, which showed on the first switch to a transcoded stream.

* don't capture the switch frame on tiles that show the camera still

frigate/api/app.py

@@ -45,7 +45,7 @@ from frigate.config.camera.updater import (
     CameraConfigUpdateEnum,
     CameraConfigUpdateTopic,
 )
-from frigate.const import REDACTED_CREDENTIAL_SENTINEL
+from frigate.const import REDACTED_CREDENTIAL_SENTINEL, REPLAY_CAMERA_PREFIX
 from frigate.ffmpeg_presets import FFMPEG_HWACCEL_VAAPI, _gpu_selector
 from frigate.genai import PROVIDERS, load_providers
 from frigate.jobs.media_sync import (
@@ -979,6 +979,17 @@ def _config_set_in_memory(request: Request, body: AppConfigSetBody) -> JSONRespo
                     status_code=400,
                 )
 
+            # an unresolvable scene would crash the threads that look up the model
+            detect_update = sections.get("detect")
+            if isinstance(detect_update, dict) and "scene" in detect_update:
+                try:
+                    config._resolve_camera_model(cam_name, detect_update["scene"])
+                except (TypeError, ValueError):
+                    return JSONResponse(
+                        content={"success": False, "message": "Invalid detect scene"},
+                        status_code=400,
+                    )
+
             for section_name, update in sections.items():
                 err = apply_section_update(camera_config, section_name, update)
                 if err is not None:
@@ -987,6 +998,11 @@ def _config_set_in_memory(request: Request, body: AppConfigSetBody) -> JSONRespo
                         status_code=400,
                     )
 
+            # the model is resolved again from the scene when the replay
+            # camera is recycled by the refresh below
+            if "detect" in sections and cam_name.startswith(REPLAY_CAMERA_PREFIX):
+                config.drop_camera_model(cam_name)
+
         # Publish ZMQ updates so processing threads pick up changes
         if body.update_topic and body.update_topic.startswith("config/cameras/"):
             _, _, camera, field = body.update_topic.split("/")

frigate/app.py

@@ -76,7 +76,7 @@ from frigate.models import (
 from frigate.notices import install_registry
 from frigate.notices.registry import NoticeRegistry
 from frigate.object_detection.base import ObjectDetectProcess
-from frigate.object_detection.util import detection_frame_size
+from frigate.object_detection.util import create_detection_input_shm
 from frigate.output.output import OutputProcess
 from frigate.ptz.autotrack import PtzAutoTracker
 from frigate.ptz.onvif import OnvifController
@@ -399,14 +399,7 @@ class FrigateApp:
             model = self.config.model_for_camera(name)
             model_cameras[model.scene].append(name)
 
-            try:
-                shm_in = UntrackedSharedMemory(
-                    name=name,
-                    create=True,
-                    size=detection_frame_size(model),
-                )
-            except FileExistsError:
-                shm_in = UntrackedSharedMemory(name=name)
+            shm_in = create_detection_input_shm(name, self.config.models)
 
             try:
                 shm_out = UntrackedSharedMemory(

frigate/camera/maintainer.py

@@ -16,7 +16,7 @@ from frigate.config.camera.updater import (
 )
 from frigate.const import REPLAY_CAMERA_PREFIX
 from frigate.models import Regions
-from frigate.object_detection.util import detection_frame_size
+from frigate.object_detection.util import create_detection_input_shm
 from frigate.util.builtin import empty_and_close_queue
 from frigate.util.image import SharedMemoryFrameManager, UntrackedSharedMemory
 from frigate.util.object import get_camera_regions_grid
@@ -127,13 +127,10 @@ class CameraMaintainer(threading.Thread):
                 max(model.width, model.height),
             )
 
+            create_detection_input_shm(name, self.config.models)
+
             try:
                 UntrackedSharedMemory(name=f"out-{name}", create=True, size=20 * 6 * 4)
-                UntrackedSharedMemory(
-                    name=name,
-                    create=True,
-                    size=detection_frame_size(model),
-                )
             except FileExistsError:
                 pass
 

frigate/camera/state.py

@@ -296,7 +296,8 @@ class CameraState:
         return frame_copy
 
     def finished(self, obj_id: str) -> None:
-        del self.tracked_objects[obj_id]
+        # state rebuilt after a resolution or scene change no longer has the object
+        self.tracked_objects.pop(obj_id, None)
 
     def on(self, event_type: str, callback: Callable[..., Any]) -> None:
         self.callbacks[event_type].append(callback)

frigate/object_detection/base.py

@@ -113,6 +113,23 @@ class AsyncLocalObjectDetector(BaseLocalDetector):
         return self.detect_api.receive_output()
 
 
+def map_input_frame(
+    frame_manager: SharedMemoryFrameManager, camera: str, model: ModelConfig
+) -> np.ndarray | None:
+    """Map a camera's detection input segment as this model's input tensor.
+
+    The segment is sized for the largest configured model, so it can be larger
+    than this model's input.
+    """
+    buf = frame_manager.write(camera)
+    shape = (1, model.height, model.width, 3)
+
+    if buf is None or len(buf) < model.height * model.width * 3:
+        return None
+
+    return np.ndarray(shape, dtype=np.uint8, buffer=buf)
+
+
 class DetectorRunner(FrigateProcess):
     def __init__(
         self,
@@ -154,14 +171,10 @@ class DetectorRunner(FrigateProcess):
                 connection_id = self.detection_queue.get(timeout=1)
             except queue.Empty:
                 continue
-            input_frame = frame_manager.get(
+            input_frame = map_input_frame(
+                frame_manager,
                 connection_id,
-                (
-                    1,
-                    self.detector_config.model.height,  # type: ignore[union-attr]
-                    self.detector_config.model.width,  # type: ignore[union-attr]
-                    3,
-                ),
+                self.detector_config.model,  # type: ignore[arg-type]
             )
 
             if input_frame is None:
@@ -227,14 +240,10 @@ class AsyncDetectorRunner(FrigateProcess):
                 continue
 
             assert self._frame_manager is not None
-            input_frame = self._frame_manager.get(
+            input_frame = map_input_frame(
+                self._frame_manager,
                 connection_id,
-                (
-                    1,
-                    self.detector_config.model.height,  # type: ignore[union-attr]
-                    self.detector_config.model.width,  # type: ignore[union-attr]
-                    3,
-                ),
+                self.detector_config.model,  # type: ignore[arg-type]
             )
 
             if input_frame is None:

frigate/object_detection/util.py

@@ -6,18 +6,39 @@ import threading
 from numpy import ndarray
 
 from frigate.detectors.detector_config import InputTensorEnum, ModelConfig
+from frigate.util.image import UntrackedSharedMemory
 
 
-def detection_frame_size(model: ModelConfig) -> int:
-    """Get the shared memory size a camera needs to hand frames to a model.
+def create_detection_input_shm(
+    camera: str, models: list[ModelConfig]
+) -> UntrackedSharedMemory:
+    """Create the shared memory a camera hands detection frames through.
+
+    It is sized for the largest model so a camera re-added under the same name
+    with another scene can reuse it. Detectors keep mappings of it, so it is
+    never unlinked while they run.
 
     Args:
-        model: The model the camera runs on
+        camera: Name of the camera
+        models: Every configured model
 
     Returns:
-        Size in bytes of one model input frame
+        The camera's detection input segment
     """
-    return model.height * model.width * 3
+    size = max(model.height * model.width * 3 for model in models)
+
+    try:
+        return UntrackedSharedMemory(name=camera, create=True, size=size)
+    except FileExistsError:
+        shm = UntrackedSharedMemory(name=camera)
+
+    if shm.size >= size:
+        return shm
+
+    # left by an earlier run with smaller models, so no detector maps it yet
+    shm.close()
+    shm.unlink()
+    return UntrackedSharedMemory(name=camera, create=True, size=size)
 
 
 class RequestStore:

frigate/test/http_api/test_http_app.py

@@ -98,6 +98,46 @@ class TestHttpApp(BaseTestHttp):
             assert response.status_code == 200
             assert app.frigate_config.cameras["front_door"].objects.track == ["person"]
 
+    @patch("frigate.detectors.detector_config.load_labels")
+    def test_config_set_in_memory_rebinds_replay_model_on_scene_change(
+        self, mock_labels
+    ):
+        mock_labels.return_value = {}
+        replay = "_replay_front_door"
+        self.minimal_config["models"] = [
+            {"devices": ["cpu"]},
+            {"scene": "thermal", "devices": ["openvino:CPU"], "width": 300},
+        ]
+        self.minimal_config["cameras"][replay] = self.minimal_config["cameras"].pop(
+            "front_door"
+        )
+        app = super().create_app()
+        app.config_publisher = Mock()
+        assert app.frigate_config.model_for_camera(replay).scene == "default"
+
+        with AuthTestClient(app) as client:
+            response = client.put(
+                "/config/set",
+                json={
+                    "requires_restart": 0,
+                    "skip_save": True,
+                    "update_topic": f"config/cameras/{replay}/detect",
+                    "config_data": {
+                        "cameras": {replay: {"detect": {"scene": "thermal"}}}
+                    },
+                },
+            )
+
+        assert response.status_code == 200
+        assert app.frigate_config.cameras[replay].detect.scene == "thermal"
+        assert app.frigate_config.model_for_camera(replay).scene == "thermal"
+        topics = [
+            call.args[0].update_type.name
+            for call in app.config_publisher.publish_update.call_args_list
+        ]
+        assert topics[0] == "detect"
+        assert topics[-1] == "refresh"
+
     ####################################################################################################################
     ###################################  Credential redaction sentinel  ################################################
     ####################################################################################################################

frigate/track/object_processing.py

@@ -91,6 +91,7 @@ class TrackedObjectProcessor(threading.Thread):
                 CameraConfigUpdateEnum.enabled,
                 CameraConfigUpdateEnum.motion,
                 CameraConfigUpdateEnum.objects,
+                CameraConfigUpdateEnum.refresh,
                 CameraConfigUpdateEnum.remove,
                 CameraConfigUpdateEnum.timestamp_style,
                 CameraConfigUpdateEnum.zones,
@@ -784,6 +785,21 @@ class TrackedObjectProcessor(threading.Thread):
 
                 self.requestor.send_data(UPDATE_CAMERA_ACTIVITY, self.camera_activity)
 
+            # a replay camera recycled onto another scene's model needs new state
+            for camera in updated_topics.get("refresh", []):
+                camera_state = self.camera_states.get(camera)
+
+                if (
+                    camera_state is None
+                    or camera_state.model is self.config.model_for_camera(camera)
+                ):
+                    continue
+
+                self.force_end_all_events(camera, camera_state)
+                camera_state.shutdown()
+                self.frame_manager.close_camera(camera)
+                self.create_camera_state(camera)
+
             # manage camera disabled state
             for camera, config in self.config.cameras.items():
                 if not config.enabled_in_config:

web/src/components/config-form/section-configs/detect.ts

@@ -240,14 +240,14 @@ const detect: SectionConfigOverrides = {
   },
   replay: {
     restartRequired: [],
-    fieldOrder: ["width", "height", "fps"],
+    fieldOrder: ["scene", "width", "height", "fps"],
     fieldGroups: {
       resolution: ["width", "height", "fps"],
     },
     hiddenFields: [
       "enabled",
       "enabled_in_config",
-      "scene",
+      hideSceneWithoutChoice,
       "min_initialized",
       "max_disappeared",
       "annotation_offset",