Implement semantic query for chat (#23206)

* sync filter entries with track and listen labels

- Auto-populate `audio.filters` from `audio.listen` instead of the full audio labelmap, matching how `objects.filters` is keyed by `track` (no longer need to populate the full audio labelmap, which was added in #22630)
- Synthesize the matching filter entries in the settings form on load so each track/listen label shows its collapsible after a profile is selected, since the backend's auto-populate only runs at config init

* translate main label for lifecycle description with attribute

* reject restricted go2rtc stream sources when added via api

* add env var check function

docker/main/rootfs/usr/local/go2rtc/create_config.py

@@ -3,7 +3,6 @@
 import json
 import os
 import sys
-from pathlib import Path
 from typing import Any
 
 from ruamel.yaml import YAML
@@ -18,37 +17,12 @@ from frigate.const import (
 )
 from frigate.ffmpeg_presets import parse_preset_hardware_acceleration_encode
 from frigate.util.config import find_config_file
+from frigate.util.services import is_restricted_go2rtc_source
 
 sys.path.remove("/opt/frigate")
 
 yaml = YAML()
 
-# Check if arbitrary exec sources are allowed (defaults to False for security)
-allow_arbitrary_exec = None
-if "GO2RTC_ALLOW_ARBITRARY_EXEC" in os.environ:
-    allow_arbitrary_exec = os.environ.get("GO2RTC_ALLOW_ARBITRARY_EXEC")
-elif (
-    os.path.isdir("/run/secrets")
-    and os.access("/run/secrets", os.R_OK)
-    and "GO2RTC_ALLOW_ARBITRARY_EXEC" in os.listdir("/run/secrets")
-):
-    allow_arbitrary_exec = (
-        Path(os.path.join("/run/secrets", "GO2RTC_ALLOW_ARBITRARY_EXEC"))
-        .read_text()
-        .strip()
-    )
-# check for the add-on options file
-elif os.path.isfile("/data/options.json"):
-    with open("/data/options.json") as f:
-        raw_options = f.read()
-    options = json.loads(raw_options)
-    allow_arbitrary_exec = options.get("go2rtc_allow_arbitrary_exec")
-
-ALLOW_ARBITRARY_EXEC = allow_arbitrary_exec is not None and str(
-    allow_arbitrary_exec
-).lower() in ("true", "1", "yes")
-
-
 config_file = find_config_file()
 
 try:
@@ -128,18 +102,13 @@ if LIBAVFORMAT_VERSION_MAJOR < 59:
         go2rtc_config["ffmpeg"]["rtsp"] = rtsp_args
 
 
-def is_restricted_source(stream_source: str) -> bool:
-    """Check if a stream source is restricted (echo, expr, or exec)."""
-    return stream_source.strip().startswith(("echo:", "expr:", "exec:"))
-
-
 for name in list(go2rtc_config.get("streams", {})):
     stream = go2rtc_config["streams"][name]
 
     if isinstance(stream, str):
         try:
             formatted_stream = substitute_frigate_vars(stream)
-            if not ALLOW_ARBITRARY_EXEC and is_restricted_source(formatted_stream):
+            if is_restricted_go2rtc_source(formatted_stream):
                 print(
                     f"[ERROR] Stream '{name}' uses a restricted source (echo/expr/exec) which is disabled by default for security. "
                     f"Set GO2RTC_ALLOW_ARBITRARY_EXEC=true to enable arbitrary exec sources."
@@ -158,7 +127,7 @@ for name in list(go2rtc_config.get("streams", {})):
         for i, stream_item in enumerate(stream):
             try:
                 formatted_stream = substitute_frigate_vars(stream_item)
-                if not ALLOW_ARBITRARY_EXEC and is_restricted_source(formatted_stream):
+                if is_restricted_go2rtc_source(formatted_stream):
                     print(
                         f"[ERROR] Stream '{name}' item {i + 1} uses a restricted source (echo/expr/exec) which is disabled by default for security. "
                         f"Set GO2RTC_ALLOW_ARBITRARY_EXEC=true to enable arbitrary exec sources."

frigate/api/camera.py

@@ -38,7 +38,7 @@ from frigate.util.builtin import clean_camera_user_pass
 from frigate.util.camera_cleanup import cleanup_camera_db, cleanup_camera_files
 from frigate.util.config import find_config_file
 from frigate.util.image import run_ffmpeg_snapshot
-from frigate.util.services import ffprobe_stream
+from frigate.util.services import ffprobe_stream, is_restricted_go2rtc_source
 
 logger = logging.getLogger(__name__)
 
@@ -147,9 +147,24 @@ def go2rtc_add_stream(request: Request, stream_name: str, src: str = ""):
         params = {"name": stream_name}
         if src:
             try:
-                params["src"] = substitute_frigate_vars(src)
+                resolved_src = substitute_frigate_vars(src)
             except KeyError:
-                params["src"] = src
+                resolved_src = src
+
+            if is_restricted_go2rtc_source(resolved_src):
+                logger.warning(
+                    "Rejected go2rtc stream '%s' with restricted source type (echo/expr/exec)",
+                    stream_name,
+                )
+                return JSONResponse(
+                    content={
+                        "success": False,
+                        "message": "Restricted stream source type",
+                    },
+                    status_code=400,
+                )
+
+            params["src"] = resolved_src
 
         r = requests.put(
             "http://127.0.0.1:1984/api/streams",

frigate/api/chat.py

@@ -68,62 +68,123 @@ class VLMMonitorRequest(BaseModel):
     zones: List[str] = []
 
 
-def get_tool_definitions() -> List[Dict[str, Any]]:
+def get_tool_definitions(
+    semantic_search_enabled: bool = False,
+) -> List[Dict[str, Any]]:
     """
     Get OpenAI-compatible tool definitions for Frigate.
 
     Returns a list of tool definitions that can be used with OpenAI-compatible
-    function calling APIs.
+    function calling APIs. When semantic search is enabled, the search_objects
+    tool exposes an additional `semantic_query` parameter for descriptive
+    queries (e.g. "person riding a lawn mower") and find_similar_objects is
+    included.
     """
+    search_objects_properties: Dict[str, Any] = {
+        "camera": {
+            "type": "string",
+            "description": "Camera name to filter by (optional).",
+        },
+        "label": {
+            "type": "string",
+            "description": (
+                "Generic object class to filter by — one of the tracked detector "
+                "labels such as 'person', 'package', 'car', 'dog', 'bird'. Use "
+                "this for broad queries like 'show me all cars today'. Combine "
+                "with semantic_query when the user also describes appearance or "
+                "behavior (e.g. label='person', semantic_query='riding a lawn "
+                "mower')."
+            ),
+        },
+        "sub_label": {
+            "type": "string",
+            "description": (
+                "Filter by a DISCRETE NAMED entity recognized in the detection. "
+                "Use this for: a known person's name ('John'), a delivery "
+                "company ('Amazon', 'UPS'), a recognized animal species or "
+                "breed ('blue jay', 'cardinal', 'golden retriever'), or a "
+                "license plate string. When filtering by a specific name, set "
+                "only sub_label and leave label unset. Do NOT use sub_label "
+                "for descriptions of appearance, clothing, or actions — those "
+                "belong in semantic_query."
+            ),
+        },
+        "after": {
+            "type": "string",
+            "description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
+        },
+        "before": {
+            "type": "string",
+            "description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
+        },
+        "zones": {
+            "type": "array",
+            "items": {"type": "string"},
+            "description": "List of zone names to filter by.",
+        },
+        "limit": {
+            "type": "integer",
+            "description": "Maximum number of objects to return (default: 25).",
+            "default": 25,
+        },
+    }
+
+    if semantic_search_enabled:
+        search_objects_properties["semantic_query"] = {
+            "type": "string",
+            "description": (
+                "Optional natural-language description of a PHYSICAL "
+                "CHARACTERISTIC, APPEARANCE, or ACTIVITY the user mentioned, "
+                "used to semantically narrow results. Only set this when the "
+                "user describes something beyond what label and sub_label can "
+                "express on their own.\n"
+                "USE for descriptive phrases like: 'riding a lawn mower', "
+                "'wearing a red jacket', 'carrying a package', 'walking a "
+                "dog', 'on a bicycle', 'holding an umbrella'.\n"
+                "DO NOT USE for:\n"
+                "- specific named people, pets, or delivery companies → use sub_label\n"
+                "- animal species or breed names like 'blue jay', 'cardinal', "
+                "'golden retriever' → use sub_label\n"
+                "- license plate strings → use sub_label\n"
+                "- generic object queries like 'all cars today' or 'every "
+                "person' → use label alone with no semantic_query\n"
+                "When set, combine with label/time/camera/zone filters as "
+                "usual (e.g. label='person', semantic_query='riding a lawn "
+                "mower', after='2024-05-01T00:00:00Z')."
+            ),
+        }
+
+    search_objects_description = (
+        "Search the historical record of detected objects in Frigate. "
+        "Use this ONLY for questions about the PAST — e.g. 'did anyone come by today?', "
+        "'when was the last car?', 'show me detections from yesterday'. "
+        "Do NOT use this for monitoring or alerting requests about future events — "
+        "use start_camera_watch instead for those. "
+        "An 'object' in Frigate represents a tracked detection (e.g., a person, package, car).\n\n"
+        "Choose filters based on what the user is asking for:\n"
+        "- Generic class query ('show me all cars today'): set `label` only.\n"
+        "- Specific NAMED entity (known person, delivery company, animal "
+        "species/breed like 'blue jay' or 'golden retriever', license "
+        "plate): set `sub_label` only and leave `label` unset.\n"
+    )
+    if semantic_search_enabled:
+        search_objects_description += (
+            "- Physical CHARACTERISTIC, APPEARANCE, or ACTIVITY that is not a "
+            "discrete name ('person riding a lawn mower', 'someone in a red "
+            "jacket', 'person carrying a package'): set `semantic_query` with "
+            "the descriptive phrase, optionally alongside `label` for the "
+            "object class. Do NOT put descriptive phrases in sub_label."
+        )
+
     return [
         {
             "type": "function",
             "function": {
                 "name": "search_objects",
-                "description": (
+                "description": search_objects_description,
-                    "Search the historical record of detected objects in Frigate. "
-                    "Use this ONLY for questions about the PAST — e.g. 'did anyone come by today?', "
-                    "'when was the last car?', 'show me detections from yesterday'. "
-                    "Do NOT use this for monitoring or alerting requests about future events — "
-                    "use start_camera_watch instead for those. "
-                    "An 'object' in Frigate represents a tracked detection (e.g., a person, package, car). "
-                    "When the user asks about a specific name (person, delivery company, animal, etc.), "
-                    "filter by sub_label only and do not set label."
-                ),
                 "parameters": {
                     "type": "object",
-                    "properties": {
+                    "properties": search_objects_properties,
-                        "camera": {
-                            "type": "string",
-                            "description": "Camera name to filter by (optional).",
-                        },
-                        "label": {
-                            "type": "string",
-                            "description": "Object label to filter by (e.g., 'person', 'package', 'car').",
-                        },
-                        "sub_label": {
-                            "type": "string",
-                            "description": "Name of a person, delivery company, animal, etc. When filtering by a specific name, use only sub_label; do not set label.",
-                        },
-                        "after": {
-                            "type": "string",
-                            "description": "Start time in ISO 8601 format (e.g., '2024-01-01T00:00:00Z').",
-                        },
-                        "before": {
-                            "type": "string",
-                            "description": "End time in ISO 8601 format (e.g., '2024-01-01T23:59:59Z').",
-                        },
-                        "zones": {
-                            "type": "array",
-                            "items": {"type": "string"},
-                            "description": "List of zone names to filter by.",
-                        },
-                        "limit": {
-                            "type": "integer",
-                            "description": "Maximum number of objects to return (default: 25).",
-                            "default": 25,
-                        },
-                    },
                 },
                 "required": [],
             },
@@ -397,9 +458,12 @@ def get_tool_definitions() -> List[Dict[str, Any]]:
     summary="Get available tools",
     description="Returns OpenAI-compatible tool definitions for function calling.",
 )
-def get_tools() -> JSONResponse:
+def get_tools(request: Request) -> JSONResponse:
     """Get list of available tools for LLM function calling."""
-    tools = get_tool_definitions()
+    semantic_search_enabled = bool(
+        getattr(request.app.frigate_config.semantic_search, "enabled", False)
+    )
+    tools = get_tool_definitions(semantic_search_enabled=semantic_search_enabled)
     return JSONResponse(content={"tools": tools})
 
 
@@ -432,16 +496,29 @@ def _resolve_zones(
 
 
 async def _execute_search_objects(
+    request: Request,
     arguments: Dict[str, Any],
     allowed_cameras: List[str],
-    config: FrigateConfig,
 ) -> JSONResponse:
     """
     Execute the search_objects tool.
 
-    This searches for detected objects (events) in Frigate using the same
+    Routes to the semantic path when the LLM supplied a `semantic_query`
-    logic as the events API endpoint.
+    and semantic search is enabled; otherwise delegates to the standard
+    events API logic.
     """
+    config = request.app.frigate_config
+    semantic_query = arguments.get("semantic_query")
+    if isinstance(semantic_query, str):
+        semantic_query = semantic_query.strip() or None
+    else:
+        semantic_query = None
+
+    if semantic_query and getattr(config.semantic_search, "enabled", False):
+        return await _execute_search_objects_semantic(
+            request, arguments, allowed_cameras, semantic_query
+        )
+
     # Parse after/before as server local time; convert to Unix timestamp
     after = arguments.get("after")
     before = arguments.get("before")
@@ -508,6 +585,119 @@ async def _execute_search_objects(
         )
 
 
+async def _execute_search_objects_semantic(
+    request: Request,
+    arguments: Dict[str, Any],
+    allowed_cameras: List[str],
+    semantic_query: str,
+) -> JSONResponse:
+    """Search objects via fused thumbnail + description embeddings.
+
+    Runs both visual and description vec searches against `semantic_query`,
+    intersects the candidates with the structured filters (camera, label,
+    sub_label, zones, time window) the LLM supplied, and ranks the survivors
+    by fused similarity. Mirrors the candidate-then-filter pattern used by
+    find_similar_objects since sqlite-vec's IN filter is unreliable.
+    """
+    from peewee import fn
+
+    config = request.app.frigate_config
+    context = request.app.embeddings
+    if context is None:
+        logger.warning(
+            "semantic_query supplied but embeddings context is unavailable; "
+            "returning empty results."
+        )
+        return JSONResponse(content=[])
+
+    after = parse_iso_to_timestamp(arguments.get("after"))
+    before = parse_iso_to_timestamp(arguments.get("before"))
+
+    camera_arg = arguments.get("camera")
+    if camera_arg and camera_arg != "all":
+        if camera_arg not in allowed_cameras:
+            return JSONResponse(content=[])
+        cameras = [camera_arg]
+    else:
+        cameras = list(allowed_cameras) if allowed_cameras else []
+
+    if not cameras:
+        return JSONResponse(content=[])
+
+    label = arguments.get("label")
+    sub_label = arguments.get("sub_label")
+
+    zones = arguments.get("zones")
+    if isinstance(zones, list) and zones:
+        zones = _resolve_zones(zones, config, cameras)
+    else:
+        zones = None
+
+    limit = int(arguments.get("limit", 25))
+    limit = max(1, min(limit, 100))
+
+    visual_distances: Dict[str, float] = {}
+    description_distances: Dict[str, float] = {}
+    try:
+        rows = context.search_thumbnail(semantic_query)
+        visual_distances = {row[0]: row[1] for row in rows}
+    except Exception:
+        logger.exception(
+            "search_thumbnail failed for semantic_query: %s", semantic_query
+        )
+
+    try:
+        rows = context.search_description(semantic_query)
+        description_distances = {row[0]: row[1] for row in rows}
+    except Exception:
+        logger.exception(
+            "search_description failed for semantic_query: %s", semantic_query
+        )
+
+    vec_ids = set(visual_distances) | set(description_distances)
+    if not vec_ids:
+        return JSONResponse(content=[])
+
+    clauses = [Event.id.in_(list(vec_ids)), Event.camera.in_(cameras)]
+    if after is not None:
+        clauses.append(Event.start_time >= after)
+    if before is not None:
+        clauses.append(Event.start_time <= before)
+    if label:
+        clauses.append(Event.label == label)
+    if sub_label:
+        # case-insensitive match to mirror events() behavior
+        clauses.append(fn.LOWER(Event.sub_label.cast("text")) == sub_label.lower())
+    if zones:
+        zone_clauses = [Event.zones.cast("text") % f'*"{zone}"*' for zone in zones]
+        clauses.append(reduce(operator.or_, zone_clauses))
+
+    eligible = {e.id: e for e in Event.select().where(reduce(operator.and_, clauses))}
+
+    scored: List[tuple[str, float]] = []
+    for eid in eligible:
+        v_score = (
+            distance_to_score(visual_distances[eid], context.thumb_stats)
+            if eid in visual_distances
+            else None
+        )
+        d_score = (
+            distance_to_score(description_distances[eid], context.desc_stats)
+            if eid in description_distances
+            else None
+        )
+        fused = fuse_scores(v_score, d_score)
+        if fused is None:
+            continue
+        scored.append((eid, fused))
+
+    scored.sort(key=lambda pair: pair[1], reverse=True)
+    scored = scored[:limit]
+
+    results = [hydrate_event(eligible[eid], score=score) for eid, score in scored]
+    return JSONResponse(content=results)
+
+
 async def _execute_find_similar_objects(
     request: Request,
     arguments: Dict[str, Any],
@@ -696,9 +886,7 @@ async def execute_tool(
     logger.debug(f"Executing tool: {tool_name} with arguments: {arguments}")
 
     if tool_name == "search_objects":
-        return await _execute_search_objects(
+        return await _execute_search_objects(request, arguments, allowed_cameras)
-            arguments, allowed_cameras, request.app.frigate_config
-        )
 
     if tool_name == "find_similar_objects":
         result = await _execute_find_similar_objects(
@@ -878,9 +1066,7 @@ async def _execute_tool_internal(
     This is used by the chat completion endpoint to execute tools.
     """
     if tool_name == "search_objects":
-        response = await _execute_search_objects(
+        response = await _execute_search_objects(request, arguments, allowed_cameras)
-            arguments, allowed_cameras, request.app.frigate_config
-        )
         try:
             if hasattr(response, "body"):
                 body_str = response.body.decode("utf-8")
@@ -1293,7 +1479,9 @@ async def chat_completion(
             status_code=400,
         )
 
-    tools = get_tool_definitions()
+    config = request.app.frigate_config
+    semantic_search_enabled = bool(getattr(config.semantic_search, "enabled", False))
+    tools = get_tool_definitions(semantic_search_enabled=semantic_search_enabled)
     conversation = []
 
     current_datetime = datetime.now()
@@ -1301,7 +1489,6 @@ async def chat_completion(
     current_time_str = current_datetime.strftime("%I:%M:%S %p")
 
     cameras_info = []
-    config = request.app.frigate_config
     has_speed_zone = False
     for camera_id in allowed_cameras:
         if camera_id not in config.cameras:
@@ -1339,6 +1526,15 @@ async def chat_completion(
         )
         speed_units_section = f"\n\nReport object speeds to the user in {speed_unit}."
 
+    semantic_search_section = ""
+    if semantic_search_enabled:
+        semantic_search_section = (
+            "\n\nWhen routing a search_objects call, pick filters by the shape of the user's request:\n"
+            "- Generic class ('show me all cars today'): set `label` only.\n"
+            "- Specific named entity — a known person ('John'), delivery company ('Amazon'), animal species/breed ('blue jay', 'cardinal', 'golden retriever'), or license plate: set `sub_label` only and leave `label` unset.\n"
+            "- Physical characteristic, appearance, or activity that is NOT a discrete name ('find me people riding a lawn mower', 'someone in a red jacket', 'a person carrying a package'): set `semantic_query` with the descriptive phrase, optionally combined with `label` for the object class. Never put descriptive phrases in `sub_label`."
+        )
+
     system_prompt = f"""You are a helpful assistant for Frigate, a security camera NVR system. You help users answer questions about their cameras, detected objects, and events.
 
 Current server local date and time: {current_date_str} at {current_time_str}
@@ -1350,7 +1546,7 @@ When users ask about "today", "yesterday", "this week", etc., use the current da
 When searching for objects or events, use ISO 8601 format for dates (e.g., {current_date_str}T00:00:00Z for the start of today).
 Always be accurate with time calculations based on the current date provided.
 
-When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:<id>], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{cameras_section}{speed_units_section}"""
+When a user refers to a specific object they have seen or describe with identifying details ("that green car", "the person in the red jacket", "a package left today"), prefer the find_similar_objects tool over search_objects. Use search_objects first only to locate the anchor event, then pass its id to find_similar_objects. For generic queries like "show me all cars today", keep using search_objects. If a user message begins with [attached_event:<id>], treat that event id as the anchor for any similarity or "tell me more" request in the same message and call find_similar_objects with that id.{semantic_search_section}{cameras_section}{speed_units_section}"""
 
     conversation.append(
         {

frigate/config/config.py

@@ -26,7 +26,6 @@ from frigate.plus import PlusApi
 from frigate.util.builtin import (
     deep_merge,
     get_ffmpeg_arg_list,
-    load_labels,
 )
 from frigate.util.config import (
     CURRENT_CONFIG_VERSION,
@@ -638,17 +637,12 @@ class FrigateConfig(FrigateBaseModel):
         if self.ffmpeg.hwaccel_args == "auto":
             self.ffmpeg.hwaccel_args = auto_detect_hwaccel()
 
-        # Populate global audio filters for all audio labels
+        # Populate global audio filters from listen. Existing user-defined
-        all_audio_labels = {
+        # entries for labels not in listen are preserved but unused at runtime.
-            label
-            for label in load_labels("/audio-labelmap.txt", prefill=521).values()
-            if label
-        }
-
         if self.audio.filters is None:
             self.audio.filters = {}
 
-        for key in sorted(all_audio_labels - self.audio.filters.keys()):
+        for key in sorted(set(self.audio.listen) - self.audio.filters.keys()):
             self.audio.filters[key] = AudioFilterConfig()
 
         self.audio.filters = dict(sorted(self.audio.filters.items()))
@@ -840,7 +834,9 @@ class FrigateConfig(FrigateBaseModel):
             if camera_config.audio.filters is None:
                 camera_config.audio.filters = {}
 
-            for key in sorted(all_audio_labels - camera_config.audio.filters.keys()):
+            for key in sorted(
+                set(camera_config.audio.listen) - camera_config.audio.filters.keys()
+            ):
                 camera_config.audio.filters[key] = AudioFilterConfig()
 
             camera_config.audio.filters = dict(

frigate/test/http_api/test_http_camera_access.py

@@ -1,3 +1,4 @@
+import os
 from unittest.mock import patch
 
 from fastapi import HTTPException, Request
@@ -357,6 +358,51 @@ class TestGo2rtcStreamAccess(BaseTestHttp):
             f"got {resp.status_code}"
         )
 
+    def test_add_stream_rejects_restricted_source(self):
+        """PUT /go2rtc/streams must reject exec:/echo:/expr: sources even for
+        admins"""
+        app = self._make_app(_MULTI_CAMERA_CONFIG)
+        with AuthTestClient(app) as client:
+            for src in (
+                "exec:/tmp/rev.sh",
+                "echo:foo",
+                "expr:bar",
+                "  exec:/tmp/rev.sh",
+            ):
+                resp = client.put(f"/go2rtc/streams/revshell?src={src}")
+                assert resp.status_code == 400, (
+                    f"Expected 400 for restricted src {src!r}; got {resp.status_code}"
+                )
+                assert resp.json().get("success") is False
+
+    def test_add_stream_allows_non_restricted_source(self):
+        """A normal stream URL should pass the restricted-source check and reach
+        the (unavailable in tests) go2rtc proxy — so we expect 500, not 400."""
+        app = self._make_app(_MULTI_CAMERA_CONFIG)
+        with AuthTestClient(app) as client:
+            resp = client.put("/go2rtc/streams/legit?src=rtsp://10.0.0.1:554/video")
+            assert resp.status_code != 400, (
+                f"Non-restricted source should not be rejected with 400; got {resp.status_code}"
+            )
+
+    def test_add_stream_allows_restricted_source_when_override_set(self):
+        """When GO2RTC_ALLOW_ARBITRARY_EXEC is set, the API must defer to operator
+        intent and forward the request to go2rtc instead of short-circuiting with 400."""
+        app = self._make_app(_MULTI_CAMERA_CONFIG)
+        mock_response = type("R", (), {"ok": True, "status_code": 200, "text": "ok"})()
+        with patch.dict(os.environ, {"GO2RTC_ALLOW_ARBITRARY_EXEC": "true"}):
+            with patch(
+                "frigate.api.camera.requests.put", return_value=mock_response
+            ) as mock_put:
+                with AuthTestClient(app) as client:
+                    resp = client.put("/go2rtc/streams/legit?src=exec:/tmp/something")
+                assert resp.status_code == 200, (
+                    f"Restricted src should be forwarded when override set; got {resp.status_code}"
+                )
+                mock_put.assert_called_once()
+                forwarded_src = mock_put.call_args.kwargs["params"]["src"]
+                assert forwarded_src == "exec:/tmp/something"
+
     def test_stream_alias_blocked_when_owning_camera_disallowed(self):
         """limited_user cannot access a stream alias that belongs to a camera they
         are not allowed to see."""

frigate/test/test_config.py

@@ -10,7 +10,7 @@ from ruamel.yaml.constructor import DuplicateKeyError
 from frigate.config import BirdseyeModeEnum, FrigateConfig
 from frigate.const import MODEL_CACHE_DIR
 from frigate.detectors import DetectorTypeEnum
-from frigate.util.builtin import deep_merge, load_labels
+from frigate.util.builtin import deep_merge
 
 
 class TestConfig(unittest.TestCase):
@@ -309,16 +309,11 @@ class TestConfig(unittest.TestCase):
         }
 
         frigate_config = FrigateConfig(**config)
-        all_audio_labels = {
+        assert set(frigate_config.cameras["back"].audio.filters.keys()) == {
-            label
+            "speech",
-            for label in load_labels("/audio-labelmap.txt", prefill=521).values()
+            "yell",
-            if label
         }
 
-        assert all_audio_labels.issubset(
-            set(frigate_config.cameras["back"].audio.filters.keys())
-        )
-
     def test_override_audio_filters(self):
         config = {
             "mqtt": {"host": "mqtt"},
@@ -345,7 +340,8 @@ class TestConfig(unittest.TestCase):
         frigate_config = FrigateConfig(**config)
         assert "speech" in frigate_config.cameras["back"].audio.filters
         assert frigate_config.cameras["back"].audio.filters["speech"].threshold == 0.9
-        assert "babbling" in frigate_config.cameras["back"].audio.filters
+        assert "yell" in frigate_config.cameras["back"].audio.filters
+        assert "babbling" not in frigate_config.cameras["back"].audio.filters
 
     def test_inherit_object_filters(self):
         config = {

frigate/util/services.py

@@ -778,6 +778,41 @@ def get_hailo_temps() -> dict[str, float]:
     return temps
 
 
+def _go2rtc_arbitrary_exec_allowed() -> bool:
+    """Read the GO2RTC_ALLOW_ARBITRARY_EXEC override from env, docker
+    secrets, or the Home Assistant add-on options file."""
+    raw: Optional[str] = None
+    if "GO2RTC_ALLOW_ARBITRARY_EXEC" in os.environ:
+        raw = os.environ.get("GO2RTC_ALLOW_ARBITRARY_EXEC")
+    elif (
+        os.path.isdir("/run/secrets")
+        and os.access("/run/secrets", os.R_OK)
+        and "GO2RTC_ALLOW_ARBITRARY_EXEC" in os.listdir("/run/secrets")
+    ):
+        try:
+            with open("/run/secrets/GO2RTC_ALLOW_ARBITRARY_EXEC") as f:
+                raw = f.read().strip()
+        except OSError:
+            raw = None
+    elif os.path.isfile("/data/options.json"):
+        try:
+            with open("/data/options.json") as f:
+                options = json.loads(f.read())
+            raw = options.get("go2rtc_allow_arbitrary_exec")
+        except (OSError, json.JSONDecodeError):
+            raw = None
+
+    return raw is not None and str(raw).lower() in ("true", "1", "yes")
+
+
+def is_restricted_go2rtc_source(stream_source: str) -> bool:
+    """Check if a stream source is a restricted type (echo, expr, or exec)
+    and the GO2RTC_ALLOW_ARBITRARY_EXEC override is not set."""
+    if not stream_source.strip().startswith(("echo:", "expr:", "exec:")):
+        return False
+    return not _go2rtc_arbitrary_exec_allowed()
+
+
 def ffprobe_stream(ffmpeg, path: str, detailed: bool = False) -> sp.CompletedProcess:
     """Run ffprobe on stream."""
     clean_path = escape_special_characters(path)

web/src/components/config-form/sections/BaseSection.tsx

@@ -22,7 +22,7 @@ import {
   modifySchemaForSection,
   getEffectiveDefaultsForSection,
   sanitizeOverridesForSection,
-  synthesizeMissingObjectFilters,
+  synthesizeMissingFilters,
 } from "./section-special-cases";
 import { getSectionValidation } from "../section-validations";
 import { useConfigOverride } from "@/hooks/use-config-override";
@@ -370,7 +370,7 @@ export function ConfigSection({
       return {};
     }
 
-    return synthesizeMissingObjectFilters(
+    return synthesizeMissingFilters(
       sectionPath,
       rawSectionValue,
       modifiedSchema ?? undefined,

web/src/components/config-form/sections/section-special-cases.ts

@@ -128,22 +128,31 @@ export function getEffectiveDefaultsForSection(
   return schemaDefaults;
 }
 
+// Sections whose `filters` dict is keyed by a sibling list field. The backend
+// auto-populates these filters at config init but doesn't re-run after profile
+// merges, so we synthesize the missing entries on the frontend.
+const FILTER_SECTIONS: Record<string, { listField: string }> = {
+  objects: { listField: "track" },
+  audio: { listField: "listen" },
+};
+
 /**
- * Add default filter entries for any label in `objects.track` that isn't
+ * Add default filter entries for any label in the section's list field
- * already in `objects.filters`, so each tracked label gets a collapsible.
+ * (e.g. `objects.track`, `audio.listen`) that isn't already in `filters`, so
- * The backend only auto-populates filters at config init, not after profile
+ * each label gets a collapsible. The backend only auto-populates filters at
- * merges.
+ * config init, not after profile merges.
  */
-export function synthesizeMissingObjectFilters(
+export function synthesizeMissingFilters(
   sectionPath: string,
   data: unknown,
   sectionSchema: RJSFSchema | undefined,
 ): unknown {
-  if (sectionPath !== "objects") return data;
+  const sectionConfig = FILTER_SECTIONS[sectionPath];
+  if (!sectionConfig) return data;
   if (!isJsonObject(data)) return data;
 
-  const trackValue = (data as JsonObject).track;
+  const listValue = (data as JsonObject)[sectionConfig.listField];
-  if (!Array.isArray(trackValue) || trackValue.length === 0) return data;
+  if (!Array.isArray(listValue) || listValue.length === 0) return data;
 
   const properties = (sectionSchema as { properties?: Record<string, unknown> })
     ?.properties;
@@ -160,7 +169,7 @@ export function synthesizeMissingObjectFilters(
 
   const newFilters: JsonObject = { ...existingFilters };
   let added = false;
-  for (const label of trackValue) {
+  for (const label of listValue) {
     if (typeof label !== "string") continue;
     if (Object.prototype.hasOwnProperty.call(newFilters, label)) continue;
     newFilters[label] = (

web/src/utils/lifecycleUtil.ts

@@ -60,7 +60,7 @@ export function getLifecycleItemDescription(
       } else {
         title = t("trackingDetails.lifecycleItemDesc.attribute.other", {
           ns: "views/explore",
-          label: lifecycleItem.data.label,
+          label: getTranslatedLabel(lifecycleItem.data.label),
           attribute: getTranslatedLabel(
             lifecycleItem.data.attribute.replaceAll("_", " "),
           ),