Improve profile state management and add recap tool (#22715)

* Improve profile information

* Add chat tools

* Add quick links to new chats

* Improve usefulness

* Cleanup

* fix

frigate/api/app.py

@@ -142,9 +142,20 @@ def config(request: Request):
     # remove the proxy secret
     config["proxy"].pop("auth_secret", None)
 
+    # remove genai api keys
+    for genai_name, genai_cfg in config.get("genai", {}).items():
+        if isinstance(genai_cfg, dict):
+            genai_cfg.pop("api_key", None)
+
     for camera_name, camera in request.app.frigate_config.cameras.items():
         camera_dict = config["cameras"][camera_name]
 
+        # remove onvif credentials
+        onvif_dict = camera_dict.get("onvif", {})
+        if onvif_dict:
+            onvif_dict.pop("user", None)
+            onvif_dict.pop("password", None)
+
         # clean paths
         for input in camera_dict.get("ffmpeg", {}).get("inputs", []):
             input["path"] = clean_camera_user_pass(input["path"])

frigate/api/chat.py

@@ -15,6 +15,7 @@ from pydantic import BaseModel
 from frigate.api.auth import (
     allow_any_authenticated,
     get_allowed_cameras_for_filter,
+    require_camera_access,
 )
 from frigate.api.defs.query.events_query_parameters import EventsQueryParams
 from frigate.api.defs.request.chat_body import ChatCompletionRequest
@@ -293,6 +294,60 @@ def get_tool_definitions() -> List[Dict[str, Any]]:
                 },
             },
         },
+        {
+            "type": "function",
+            "function": {
+                "name": "get_profile_status",
+                "description": (
+                    "Get the current profile status including the active profile and "
+                    "timestamps of when each profile was last activated. Use this to "
+                    "determine time periods for recap requests — e.g. when the user asks "
+                    "'what happened while I was away?', call this first to find the relevant "
+                    "time window based on profile activation history."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {},
+                    "required": [],
+                },
+            },
+        },
+        {
+            "type": "function",
+            "function": {
+                "name": "get_recap",
+                "description": (
+                    "Get a recap of all activity (alerts and detections) for a given time period. "
+                    "Use this after calling get_profile_status to retrieve what happened during "
+                    "a specific window — e.g. 'what happened while I was away?'. Returns a "
+                    "chronological list of activity with camera, objects, zones, and GenAI-generated "
+                    "descriptions when available. Summarize the results for the user."
+                ),
+                "parameters": {
+                    "type": "object",
+                    "properties": {
+                        "after": {
+                            "type": "string",
+                            "description": "Start of the time period in ISO 8601 format (e.g. '2025-03-15T08:00:00').",
+                        },
+                        "before": {
+                            "type": "string",
+                            "description": "End of the time period in ISO 8601 format (e.g. '2025-03-15T17:00:00').",
+                        },
+                        "cameras": {
+                            "type": "string",
+                            "description": "Comma-separated camera IDs to include, or 'all' for all cameras. Default is 'all'.",
+                        },
+                        "severity": {
+                            "type": "string",
+                            "enum": ["alert", "detection"],
+                            "description": "Filter by severity level. Omit to include both alerts and detections.",
+                        },
+                    },
+                    "required": ["after", "before"],
+                },
+            },
+        },
     ]
 
 
@@ -645,10 +700,14 @@ async def _execute_tool_internal(
         return await _execute_start_camera_watch(request, arguments)
     elif tool_name == "stop_camera_watch":
         return _execute_stop_camera_watch()
+    elif tool_name == "get_profile_status":
+        return _execute_get_profile_status(request)
+    elif tool_name == "get_recap":
+        return _execute_get_recap(arguments, allowed_cameras)
     else:
         logger.error(
             "Tool call failed: unknown tool %r. Expected one of: search_objects, get_live_context, "
-            "start_camera_watch, stop_camera_watch. Arguments received: %s",
+            "start_camera_watch, stop_camera_watch, get_profile_status, get_recap. Arguments received: %s",
             tool_name,
             json.dumps(arguments),
         )
@@ -672,6 +731,8 @@ async def _execute_start_camera_watch(
     if camera not in config.cameras:
         return {"error": f"Camera '{camera}' not found."}
 
+    await require_camera_access(camera, request=request)
+
     genai_manager = request.app.genai_manager
     vision_client = genai_manager.vision_client or genai_manager.tool_client
     if vision_client is None:
@@ -710,6 +771,168 @@ def _execute_stop_camera_watch() -> Dict[str, Any]:
     return {"success": False, "message": "No active watch job to cancel."}
 
 
+def _execute_get_profile_status(request: Request) -> Dict[str, Any]:
+    """Return profile status including active profile and activation timestamps."""
+    profile_manager = getattr(request.app, "profile_manager", None)
+    if profile_manager is None:
+        return {"error": "Profile manager is not available."}
+
+    info = profile_manager.get_profile_info()
+
+    # Convert timestamps to human-readable local times inline
+    last_activated = {}
+    for name, ts in info.get("last_activated", {}).items():
+        try:
+            dt = datetime.fromtimestamp(ts)
+            last_activated[name] = dt.strftime("%Y-%m-%d %I:%M:%S %p")
+        except (TypeError, ValueError, OSError):
+            last_activated[name] = str(ts)
+
+    return {
+        "active_profile": info.get("active_profile"),
+        "profiles": info.get("profiles", []),
+        "last_activated": last_activated,
+    }
+
+
+def _execute_get_recap(
+    arguments: Dict[str, Any],
+    allowed_cameras: List[str],
+) -> Dict[str, Any]:
+    """Fetch review segments with GenAI metadata for a time period."""
+    from functools import reduce
+
+    from peewee import operator
+
+    from frigate.models import ReviewSegment
+
+    after_str = arguments.get("after")
+    before_str = arguments.get("before")
+
+    def _parse_as_local_timestamp(s: str):
+        s = s.replace("Z", "").strip()[:19]
+        dt = datetime.strptime(s, "%Y-%m-%dT%H:%M:%S")
+        return time.mktime(dt.timetuple())
+
+    try:
+        after = _parse_as_local_timestamp(after_str)
+    except (ValueError, AttributeError, TypeError):
+        return {"error": f"Invalid 'after' timestamp: {after_str}"}
+
+    try:
+        before = _parse_as_local_timestamp(before_str)
+    except (ValueError, AttributeError, TypeError):
+        return {"error": f"Invalid 'before' timestamp: {before_str}"}
+
+    cameras = arguments.get("cameras", "all")
+    if cameras != "all":
+        requested = set(cameras.split(","))
+        camera_list = list(requested.intersection(allowed_cameras))
+        if not camera_list:
+            return {"events": [], "message": "No accessible cameras matched."}
+    else:
+        camera_list = allowed_cameras
+
+    clauses = [
+        (ReviewSegment.start_time < before)
+        & ((ReviewSegment.end_time.is_null(True)) | (ReviewSegment.end_time > after)),
+        (ReviewSegment.camera << camera_list),
+    ]
+
+    severity_filter = arguments.get("severity")
+    if severity_filter:
+        clauses.append(ReviewSegment.severity == severity_filter)
+
+    try:
+        rows = (
+            ReviewSegment.select(
+                ReviewSegment.camera,
+                ReviewSegment.start_time,
+                ReviewSegment.end_time,
+                ReviewSegment.severity,
+                ReviewSegment.data,
+            )
+            .where(reduce(operator.and_, clauses))
+            .order_by(ReviewSegment.start_time.asc())
+            .limit(100)
+            .dicts()
+            .iterator()
+        )
+
+        events: List[Dict[str, Any]] = []
+
+        for row in rows:
+            data = row.get("data") or {}
+            if isinstance(data, str):
+                try:
+                    data = json.loads(data)
+                except json.JSONDecodeError:
+                    data = {}
+
+            camera = row["camera"]
+            event: Dict[str, Any] = {
+                "camera": camera.replace("_", " ").title(),
+                "severity": row.get("severity", "detection"),
+            }
+
+            # Include GenAI metadata when available
+            metadata = data.get("metadata")
+            if metadata and isinstance(metadata, dict):
+                if metadata.get("title"):
+                    event["title"] = metadata["title"]
+                if metadata.get("scene"):
+                    event["description"] = metadata["scene"]
+                threat = metadata.get("potential_threat_level")
+                if threat is not None:
+                    threat_labels = {
+                        0: "normal",
+                        1: "needs_review",
+                        2: "security_concern",
+                    }
+                    event["threat_level"] = threat_labels.get(threat, str(threat))
+
+            # Only include objects/zones/audio when there's no GenAI description
+            # to keep the payload concise — the description already covers these
+            if "description" not in event:
+                objects = data.get("objects", [])
+                if objects:
+                    event["objects"] = objects
+                zones = data.get("zones", [])
+                if zones:
+                    event["zones"] = zones
+                audio = data.get("audio", [])
+                if audio:
+                    event["audio"] = audio
+
+            start_ts = row.get("start_time")
+            end_ts = row.get("end_time")
+            if start_ts is not None:
+                try:
+                    event["time"] = datetime.fromtimestamp(start_ts).strftime(
+                        "%I:%M %p"
+                    )
+                except (TypeError, ValueError, OSError):
+                    pass
+            if end_ts is not None and start_ts is not None:
+                try:
+                    event["duration_seconds"] = round(end_ts - start_ts)
+                except (TypeError, ValueError):
+                    pass
+
+            events.append(event)
+
+        if not events:
+            return {
+                "events": [],
+                "message": "No activity was found during this time period.",
+            }
+
+        return {"events": events}
+    except Exception as e:
+        logger.error("Error executing get_recap: %s", e, exc_info=True)
+        return {"error": "Failed to fetch recap data."}
+
+
 async def _execute_pending_tools(
     pending_tool_calls: List[Dict[str, Any]],
     request: Request,
@@ -1156,6 +1379,8 @@ async def start_vlm_monitor(
             status_code=404,
         )
 
+    await require_camera_access(body.camera, request=request)
+
     vision_client = genai_manager.vision_client or genai_manager.tool_client
     if vision_client is None:
         return JSONResponse(

frigate/api/media.py

@@ -896,6 +896,7 @@ async def event_thumbnail(
                 if event_id in camera_state.tracked_objects:
                     tracked_obj = camera_state.tracked_objects.get(event_id)
                     if tracked_obj is not None:
+                        await require_camera_access(camera_state.name, request=request)
                         thumbnail_bytes = tracked_obj.get_thumbnail(extension.value)
         except Exception:
             return JSONResponse(
@@ -1066,7 +1067,7 @@ def grid_snapshot(
 
 
 @router.delete(
-    "/{camera_name}/region_grid", dependencies=[Depends(require_role("admin"))]
+    "/{camera_name}/region_grid", dependencies=[Depends(require_role(["admin"]))]
 )
 def clear_region_grid(request: Request, camera_name: str):
     """Clear the region grid for a camera."""

frigate/api/review.py

@@ -742,7 +742,7 @@ async def set_not_reviewed(
 
 @router.post(
     "/review/summarize/start/{start_ts}/end/{end_ts}",
-    dependencies=[Depends(allow_any_authenticated())],
+    dependencies=[Depends(require_role(["admin"]))],
     description="Use GenAI to summarize review items over a period of time.",
 )
 def generate_review_summary(request: Request, start_ts: float, end_ts: float):

frigate/config/profile_manager.py

@@ -1,7 +1,9 @@
 """Profile manager for activating/deactivating named config profiles."""
 
 import copy
+import json
 import logging
+from datetime import datetime, timezone
 from pathlib import Path
 from typing import Optional
 
@@ -32,7 +34,7 @@ PROFILE_SECTION_UPDATES: dict[str, CameraConfigUpdateEnum] = {
     "zones": CameraConfigUpdateEnum.zones,
 }
 
-PERSISTENCE_FILE = Path(CONFIG_DIR) / ".active_profile"
+PERSISTENCE_FILE = Path(CONFIG_DIR) / ".profiles"
 
 
 class ProfileManager:
@@ -291,25 +293,36 @@ class ProfileManager:
                     )
 
     def _persist_active_profile(self, profile_name: Optional[str]) -> None:
-        """Persist the active profile name to disk."""
+        """Persist the active profile state to disk as JSON."""
         try:
-            if profile_name is None:
+            data = self._load_persisted_data()
-                PERSISTENCE_FILE.unlink(missing_ok=True)
+            data["active"] = profile_name
-            else:
+            if profile_name is not None:
-                PERSISTENCE_FILE.write_text(profile_name)
+                data.setdefault("last_activated", {})[profile_name] = datetime.now(
+                    timezone.utc
+                ).timestamp()
+            PERSISTENCE_FILE.write_text(json.dumps(data))
         except OSError:
             logger.exception("Failed to persist active profile")
 
     @staticmethod
-    def load_persisted_profile() -> Optional[str]:
+    def _load_persisted_data() -> dict:
-        """Load the persisted active profile name from disk."""
+        """Load the full persisted profile data from disk."""
         try:
             if PERSISTENCE_FILE.exists():
-                name = PERSISTENCE_FILE.read_text().strip()
+                raw = PERSISTENCE_FILE.read_text().strip()
+                if raw:
+                    return json.loads(raw)
+        except (OSError, json.JSONDecodeError):
+            logger.exception("Failed to load persisted profile data")
+        return {"active": None, "last_activated": {}}
+
+    @staticmethod
+    def load_persisted_profile() -> Optional[str]:
+        """Load the persisted active profile name from disk."""
+        data = ProfileManager._load_persisted_data()
+        name = data.get("active")
         return name if name else None
-        except OSError:
-            logger.exception("Failed to load persisted profile")
-        return None
 
     def get_base_configs_for_api(self, camera_name: str) -> dict[str, dict]:
         """Return base (pre-profile) section configs for a camera.
@@ -328,7 +341,9 @@ class ProfileManager:
 
     def get_profile_info(self) -> dict:
         """Get profile state info for API responses."""
+        data = self._load_persisted_data()
         return {
             "profiles": self.get_available_profiles(),
             "active_profile": self.config.active_profile,
+            "last_activated": data.get("last_activated", {}),
         }

frigate/record/export.py

@@ -36,7 +36,9 @@ logger = logging.getLogger(__name__)
 DEFAULT_TIME_LAPSE_FFMPEG_ARGS = "-vf setpts=0.04*PTS -r 30"
 TIMELAPSE_DATA_INPUT_ARGS = "-an -skip_frame nokey"
 
-# ffmpeg flags that can read from or write to arbitrary files
+# ffmpeg flags that can read from or write to arbitrary files.
+# filter flags are blocked because source filters like movie= and
+# amovie= can read arbitrary files from the filesystem.
 BLOCKED_FFMPEG_ARGS = frozenset(
     {
         "-i",
@@ -45,6 +47,12 @@ BLOCKED_FFMPEG_ARGS = frozenset(
         "-passlogfile",
         "-sdp_file",
         "-dump_attachment",
+        "-filter_complex",
+        "-lavfi",
+        "-vf",
+        "-af",
+        "-filter",
+        "-attach",
     }
 )
 

frigate/test/test_profiles.py

@@ -1,7 +1,7 @@
 """Tests for the profiles system."""
 
+import json
 import os
-import tempfile
 import unittest
 from unittest.mock import MagicMock, patch
 
@@ -549,10 +549,17 @@ class TestProfileManager(unittest.TestCase):
 
     def test_get_profile_info(self):
         """Profile info returns correct structure with friendly names."""
+        with patch.object(
+            ProfileManager,
+            "_load_persisted_data",
+            return_value={"active": None, "last_activated": {}},
+        ):
             info = self.manager.get_profile_info()
         assert "profiles" in info
         assert "active_profile" in info
+        assert "last_activated" in info
         assert info["active_profile"] is None
+        assert info["last_activated"] == {}
         names = [p["name"] for p in info["profiles"]]
         assert "armed" in names
         assert "disarmed" in names
@@ -590,33 +597,22 @@ class TestProfilePersistence(unittest.TestCase):
     """Test profile persistence to disk."""
 
     def test_persist_and_load(self):
-        """Active profile name can be persisted and loaded."""
+        """Active profile name can be persisted and loaded via JSON."""
-        with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
+        data = {"active": "armed", "last_activated": {"armed": 1700000000.0}}
-            temp_path = f.name
+        with patch.object(
-
+            ProfileManager,
-        try:
+            "_load_persisted_data",
-            from pathlib import Path
+            return_value=data,
-
+        ):
-            path = Path(temp_path)
+            result = ProfileManager.load_persisted_profile()
-            path.write_text("armed")
+            assert result == "armed"
-            loaded = path.read_text().strip()
-            assert loaded == "armed"
-        finally:
-            os.unlink(temp_path)
 
     def test_load_empty_file(self):
         """Empty persistence file returns None."""
-        with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
-            f.write("")
-            temp_path = f.name
-
-        try:
         with patch.object(type(PERSISTENCE_FILE), "exists", return_value=True):
             with patch.object(type(PERSISTENCE_FILE), "read_text", return_value=""):
                 result = ProfileManager.load_persisted_profile()
                 assert result is None
-        finally:
-            os.unlink(temp_path)
 
     def test_load_missing_file(self):
         """Missing persistence file returns None."""
@@ -624,6 +620,118 @@ class TestProfilePersistence(unittest.TestCase):
             result = ProfileManager.load_persisted_profile()
             assert result is None
 
+    def test_load_persisted_data_valid_json(self):
+        """Valid JSON file is loaded correctly."""
+        data = {"active": "home", "last_activated": {"home": 1700000000.0}}
+        with patch.object(type(PERSISTENCE_FILE), "exists", return_value=True):
+            with patch.object(
+                type(PERSISTENCE_FILE),
+                "read_text",
+                return_value=json.dumps(data),
+            ):
+                result = ProfileManager._load_persisted_data()
+                assert result == data
+
+    def test_load_persisted_data_invalid_json(self):
+        """Invalid JSON returns default structure."""
+        with patch.object(type(PERSISTENCE_FILE), "exists", return_value=True):
+            with patch.object(
+                type(PERSISTENCE_FILE), "read_text", return_value="not json"
+            ):
+                result = ProfileManager._load_persisted_data()
+                assert result == {"active": None, "last_activated": {}}
+
+    def test_load_persisted_data_missing_file(self):
+        """Missing file returns default structure."""
+        with patch.object(type(PERSISTENCE_FILE), "exists", return_value=False):
+            result = ProfileManager._load_persisted_data()
+            assert result == {"active": None, "last_activated": {}}
+
+    def test_persist_records_timestamp(self):
+        """Persisting a profile records the activation timestamp."""
+        config_data = {
+            "mqtt": {"host": "mqtt"},
+            "profiles": {"armed": {"friendly_name": "Armed"}},
+            "cameras": {
+                "front": {
+                    "ffmpeg": {
+                        "inputs": [
+                            {
+                                "path": "rtsp://10.0.0.1:554/video",
+                                "roles": ["detect"],
+                            }
+                        ]
+                    },
+                    "detect": {"height": 1080, "width": 1920, "fps": 5},
+                    "profiles": {"armed": {"detect": {"enabled": True}}},
+                },
+            },
+        }
+        if not os.path.exists(MODEL_CACHE_DIR) and not os.path.islink(MODEL_CACHE_DIR):
+            os.makedirs(MODEL_CACHE_DIR)
+        config = FrigateConfig(**config_data)
+        manager = ProfileManager(config, MagicMock())
+
+        written_data = {}
+
+        def mock_write(_self, content):
+            written_data.update(json.loads(content))
+
+        with patch.object(
+            ProfileManager,
+            "_load_persisted_data",
+            return_value={"active": None, "last_activated": {}},
+        ):
+            with patch.object(type(PERSISTENCE_FILE), "write_text", mock_write):
+                manager._persist_active_profile("armed")
+
+        assert written_data["active"] == "armed"
+        assert "armed" in written_data["last_activated"]
+        assert isinstance(written_data["last_activated"]["armed"], float)
+
+    def test_persist_deactivate_keeps_timestamps(self):
+        """Deactivating sets active to None but preserves last_activated."""
+        existing = {
+            "active": "armed",
+            "last_activated": {"armed": 1700000000.0},
+        }
+        written_data = {}
+
+        def mock_write(_self, content):
+            written_data.update(json.loads(content))
+
+        config_data = {
+            "mqtt": {"host": "mqtt"},
+            "profiles": {"armed": {"friendly_name": "Armed"}},
+            "cameras": {
+                "front": {
+                    "ffmpeg": {
+                        "inputs": [
+                            {
+                                "path": "rtsp://10.0.0.1:554/video",
+                                "roles": ["detect"],
+                            }
+                        ]
+                    },
+                    "detect": {"height": 1080, "width": 1920, "fps": 5},
+                    "profiles": {"armed": {"detect": {"enabled": True}}},
+                },
+            },
+        }
+        if not os.path.exists(MODEL_CACHE_DIR) and not os.path.islink(MODEL_CACHE_DIR):
+            os.makedirs(MODEL_CACHE_DIR)
+        config = FrigateConfig(**config_data)
+        manager = ProfileManager(config, MagicMock())
+
+        with patch.object(
+            ProfileManager, "_load_persisted_data", return_value=existing
+        ):
+            with patch.object(type(PERSISTENCE_FILE), "write_text", mock_write):
+                manager._persist_active_profile(None)
+
+        assert written_data["active"] is None
+        assert written_data["last_activated"]["armed"] == 1700000000.0
+
 
 if __name__ == "__main__":
     unittest.main()

web/public/locales/en/views/chat.json

@@ -15,10 +15,14 @@
   "suggested_requests": "Try asking:",
   "starting_requests": {
     "show_recent_events": "Show recent events",
-    "show_camera_status": "Show camera status"
+    "show_camera_status": "Show camera status",
+    "recap": "What happened while I was away?",
+    "watch_camera": "Watch a camera for activity"
   },
   "starting_requests_prompts": {
     "show_recent_events": "Show me the recent events from the last hour",
-    "show_camera_status": "What is the current status of my cameras?"
+    "show_camera_status": "What is the current status of my cameras?",
+    "recap": "What happened while I was away?",
+    "watch_camera": "Watch the front door and let me know if anyone shows up"
   }
 }

web/src/components/chat/ChatStartingState.tsx

@@ -22,6 +22,14 @@ export function ChatStartingState({ onSendMessage }: ChatStartingStateProps) {
       label: t("starting_requests.show_camera_status"),
       prompt: t("starting_requests_prompts.show_camera_status"),
     },
+    {
+      label: t("starting_requests.recap"),
+      prompt: t("starting_requests_prompts.recap"),
+    },
+    {
+      label: t("starting_requests.watch_camera"),
+      prompt: t("starting_requests_prompts.watch_camera"),
+    },
   ];
 
   const handleRequestClick = (prompt: string) => {

web/src/types/profile.ts

@@ -14,6 +14,7 @@ export type ProfileInfo = {
 export type ProfilesApiResponse = {
   profiles: ProfileInfo[];
   active_profile: string | null;
+  last_activated: Record<string, number>;
 };
 
 export type ProfileState = {

web/src/utils/configUtil.ts

@@ -649,8 +649,11 @@ const mergeSectionConfig = (
       return srcValue ?? objValue;
     }
 
-    if (key === "uiSchema" && srcValue !== undefined) {
+    if (key === "uiSchema") {
-      return srcValue;
+      if (objValue && srcValue) {
+        return merge({}, objValue, srcValue);
+      }
+      return srcValue ?? objValue;
     }
 
     return undefined;