Merge 0f3dd097ec into 01a7ec1060

* Move openai specific workaround so it doesn't apply to other providers

* Fix gemini tool calling

* Improve efficiency of frame listing for previews

* debug replay fixes

- initial selection without changing the radio button in the dialog would select 1 hour (rather than 1 minute)
- use CLIPS_DIR instead of CACHE_DIR so that longer replay clips don't cause tmpfs cache overflows

* don't re-render the tracking details overlay on every video time tick

* change pinned to planned

---------

Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com>

.github/pull_request_template.md

@@ -26,7 +26,7 @@ _Please read the [contributing guidelines](https://github.com/blakeblackshear/fr
 
 - This PR fixes or closes issue: fixes #
 - This PR is related to issue:
-- Link to discussion with maintainers (**required** for large/pinned features):
+- Link to discussion with maintainers (**required** for any large or "planned" features):
 
 ## For new features
 

.github/workflows/stale.yml

@@ -19,8 +19,8 @@ jobs:
           days-before-stale: 30
           days-before-close: 3
           exempt-draft-pr: true
-          exempt-issue-labels: "pinned,security"
-          exempt-pr-labels: "pinned,security,dependencies"
+          exempt-issue-labels: "planned,security"
+          exempt-pr-labels: "planned,security,dependencies"
           operations-per-run: 120
       - name: Print outputs
         env:

CONTRIBUTING.md

@@ -12,7 +12,7 @@ If you've found a bug and want to fix it, go for it. Link to the relevant issue
 
 Every new feature adds scope that the maintainers must test, maintain, and support long-term. Before writing code for a new feature:
 
-1. **Check for existing discussion.** Search [feature requests](https://github.com/blakeblackshear/frigate/issues) and [discussions](https://github.com/blakeblackshear/frigate/discussions) to see if it's been proposed or discussed. Pinned feature requests are on our radar — we plan to get to them, but we don't maintain a public roadmap or timeline. Check in with us first if you have interest in contributing to one.
+1. **Check for existing discussion.** Search [feature requests](https://github.com/blakeblackshear/frigate/issues) and [discussions](https://github.com/blakeblackshear/frigate/discussions) to see if it's been proposed or discussed. Feature requests tagged with "planned" are on our radar — we plan to get to them, but we don't maintain a public roadmap or timeline. Check in with us first if you have interest in contributing to one.
 2. **Start a discussion or feature request first.** This helps ensure your idea aligns with Frigate's direction before you invest time building it. Community interest in a feature request helps us gauge demand, though a great idea is a great idea even without a crowd behind it.
 3. **Be open to "no".** We try to be thoughtful about what we take on, and sometimes that means saying no to good code if the feature isn't the right fit for the project. These calls are sometimes subjective, and we won't always get them right. We're happy to discuss and reconsider.
 

docs/docs/configuration/face_recognition.md

@@ -171,7 +171,7 @@ When choosing images to include in the face training set it is recommended to al
 - If it is difficult to make out details in a persons face it will not be helpful in training.
 - Avoid images with extreme under/over-exposure.
 - Avoid blurry / pixelated images.
-- Avoid training on infrared (gray-scale). The models are trained on color images and will be able to extract features from gray-scale images.
+- Avoid training on infrared (gray-scale). The models are trained on color images and will not be able to extract features from gray-scale images.
 - Using images of people wearing hats / sunglasses may confuse the model.
 - Do not upload too many similar images at the same time, it is recommended to train no more than 4-6 similar images for each person to avoid over-fitting.
 

frigate/api/media.py

@@ -1368,12 +1368,17 @@ def preview_gif(
         file_start = f"preview_{camera_name}-"
         start_file = f"{file_start}{start_ts}.{PREVIEW_FRAME_TYPE}"
         end_file = f"{file_start}{end_ts}.{PREVIEW_FRAME_TYPE}"
+
+        camera_files = [
+            entry.name
+            for entry in os.scandir(preview_dir)
+            if entry.name.startswith(file_start)
+        ]
+        camera_files.sort()
+
         selected_previews = []
 
-        for file in sorted(os.listdir(preview_dir)):
-            if not file.startswith(file_start):
-                continue
-
+        for file in camera_files:
             if file < start_file:
                 continue
 
@@ -1550,12 +1555,17 @@ def preview_mp4(
         file_start = f"preview_{camera_name}-"
         start_file = f"{file_start}{start_ts}.{PREVIEW_FRAME_TYPE}"
         end_file = f"{file_start}{end_ts}.{PREVIEW_FRAME_TYPE}"
+
+        camera_files = [
+            entry.name
+            for entry in os.scandir(preview_dir)
+            if entry.name.startswith(file_start)
+        ]
+        camera_files.sort()
+
         selected_previews = []
 
-        for file in sorted(os.listdir(preview_dir)):
-            if not file.startswith(file_start):
-                continue
-
+        for file in camera_files:
             if file < start_file:
                 continue
 

frigate/api/preview.py

@@ -148,12 +148,17 @@ def get_preview_frames_from_cache(camera_name: str, start_ts: float, end_ts: flo
     file_start = f"preview_{camera_name}-"
     start_file = f"{file_start}{start_ts}.{PREVIEW_FRAME_TYPE}"
     end_file = f"{file_start}{end_ts}.{PREVIEW_FRAME_TYPE}"
+
+    camera_files = [
+        entry.name
+        for entry in os.scandir(preview_dir)
+        if entry.name.startswith(file_start)
+    ]
+    camera_files.sort()
+
     selected_previews = []
 
-    for file in sorted(os.listdir(preview_dir)):
-        if not file.startswith(file_start):
-            continue
-
+    for file in camera_files:
         if file < start_file:
             continue
 

frigate/const.py

@@ -15,7 +15,7 @@ TRIGGER_DIR = f"{CLIPS_DIR}/triggers"
 BIRDSEYE_PIPE = "/tmp/cache/birdseye"
 CACHE_DIR = "/tmp/cache"
 REPLAY_CAMERA_PREFIX = "_replay_"
-REPLAY_DIR = os.path.join(CACHE_DIR, "replay")
+REPLAY_DIR = os.path.join(CLIPS_DIR, "replay")
 PLUS_ENV_VAR = "PLUS_API_KEY"
 PLUS_API_HOST = "https://api.frigate.video"
 

frigate/data_processing/post/review_descriptions.py

@@ -366,12 +366,17 @@ class ReviewDescriptionProcessor(PostProcessorApi):
         file_start = f"preview_{camera}-"
         start_file = f"{file_start}{start_time}.webp"
         end_file = f"{file_start}{end_time}.webp"
+
+        camera_files = [
+            entry.name
+            for entry in os.scandir(preview_dir)
+            if entry.name.startswith(file_start)
+        ]
+        camera_files.sort()
+
         all_frames: list[str] = []
 
-        for file in sorted(os.listdir(preview_dir)):
-            if not file.startswith(file_start):
-                continue
-
+        for file in camera_files:
             if file < start_file:
                 if len(all_frames):
                     all_frames[0] = os.path.join(preview_dir, file)

frigate/embeddings/__init__.py

@@ -4,6 +4,7 @@ import base64
 import json
 import logging
 import os
+import sys
 import threading
 from json.decoder import JSONDecodeError
 from multiprocessing.synchronize import Event as MpEvent
@@ -52,6 +53,14 @@ class EmbeddingProcess(FrigateProcess):
             self.stop_event,
         )
         maintainer.start()
+        maintainer.join()
+
+        # If the maintainer thread exited but no shutdown was requested, it
+        # crashed. Surface as a non-zero exit so the watchdog restarts us
+        # instead of treating the silent thread death as a clean shutdown.
+        if not self.stop_event.is_set():
+            logger.error("Embeddings maintainer thread exited unexpectedly")
+            sys.exit(1)
 
 
 class EmbeddingsContext:

frigate/genai/__init__.py

@@ -153,9 +153,6 @@ Each line represents a detection state, not necessarily unique individuals. The
             if "other_concerns" in schema.get("required", []):
                 schema["required"].remove("other_concerns")
 
-        # OpenAI strict mode requires additionalProperties: false on all objects
-        schema["additionalProperties"] = False
-
         response_format = {
             "type": "json_schema",
             "json_schema": {

frigate/genai/gemini.py

@@ -136,22 +136,44 @@ class GeminiClient(GenAIClient):
                             )
                         )
                 elif role == "assistant":
-                    gemini_messages.append(
-                        types.Content(
-                            role="model", parts=[types.Part.from_text(text=content)]
-                        )
-                    )
+                    parts: list[types.Part] = []
+                    if content:
+                        parts.append(types.Part.from_text(text=content))
+                    for tc in msg.get("tool_calls") or []:
+                        func = tc.get("function") or {}
+                        tc_name = func.get("name") or ""
+                        tc_args: Any = func.get("arguments")
+                        if isinstance(tc_args, str):
+                            try:
+                                tc_args = json.loads(tc_args)
+                            except (json.JSONDecodeError, TypeError):
+                                tc_args = {}
+                        if not isinstance(tc_args, dict):
+                            tc_args = {}
+                        if tc_name:
+                            parts.append(
+                                types.Part.from_function_call(
+                                    name=tc_name, args=tc_args
+                                )
+                            )
+                    if not parts:
+                        parts.append(types.Part.from_text(text=" "))
+                    gemini_messages.append(types.Content(role="model", parts=parts))
                 elif role == "tool":
                     # Handle tool response
-                    function_response = {
-                        "name": msg.get("name", ""),
-                        "response": content,
-                    }
+                    response_payload = (
+                        content if isinstance(content, dict) else {"result": content}
+                    )
                     gemini_messages.append(
                         types.Content(
                             role="function",
                             parts=[
-                                types.Part.from_function_response(function_response)  # type: ignore[misc,call-arg,arg-type]
+                                types.Part.from_function_response(
+                                    name=msg.get("name")
+                                    or msg.get("tool_call_id")
+                                    or "",
+                                    response=response_payload,
+                                )
                             ],
                         )
                     )
@@ -343,22 +365,44 @@ class GeminiClient(GenAIClient):
                             )
                         )
                 elif role == "assistant":
-                    gemini_messages.append(
-                        types.Content(
-                            role="model", parts=[types.Part.from_text(text=content)]
-                        )
-                    )
+                    parts: list[types.Part] = []
+                    if content:
+                        parts.append(types.Part.from_text(text=content))
+                    for tc in msg.get("tool_calls") or []:
+                        func = tc.get("function") or {}
+                        tc_name = func.get("name") or ""
+                        tc_args: Any = func.get("arguments")
+                        if isinstance(tc_args, str):
+                            try:
+                                tc_args = json.loads(tc_args)
+                            except (json.JSONDecodeError, TypeError):
+                                tc_args = {}
+                        if not isinstance(tc_args, dict):
+                            tc_args = {}
+                        if tc_name:
+                            parts.append(
+                                types.Part.from_function_call(
+                                    name=tc_name, args=tc_args
+                                )
+                            )
+                    if not parts:
+                        parts.append(types.Part.from_text(text=" "))
+                    gemini_messages.append(types.Content(role="model", parts=parts))
                 elif role == "tool":
                     # Handle tool response
-                    function_response = {
-                        "name": msg.get("name", ""),
-                        "response": content,
-                    }
+                    response_payload = (
+                        content if isinstance(content, dict) else {"result": content}
+                    )
                     gemini_messages.append(
                         types.Content(
                             role="function",
                             parts=[
-                                types.Part.from_function_response(function_response)  # type: ignore[misc,call-arg,arg-type]
+                                types.Part.from_function_response(
+                                    name=msg.get("name")
+                                    or msg.get("tool_call_id")
+                                    or "",
+                                    response=response_payload,
+                                )
                             ],
                         )
                     )

frigate/genai/llama_cpp.py

@@ -44,6 +44,7 @@ class LlamaCppClient(GenAIClient):
     _supports_tools: bool
     _image_token_cache: dict[tuple[int, int], int]
     _text_baseline_tokens: int | None
+    _media_marker: str
 
     def _init_provider(self) -> str | None:
         """Initialize the client and query model metadata from the server."""
@@ -56,6 +57,7 @@ class LlamaCppClient(GenAIClient):
         self._supports_tools = False
         self._image_token_cache = {}
         self._text_baseline_tokens = None
+        self._media_marker = "<__media__>"
 
         base_url = (
             self.genai_config.base_url.rstrip("/")
@@ -141,6 +143,13 @@ class LlamaCppClient(GenAIClient):
             chat_caps = props.get("chat_template_caps", {})
             self._supports_tools = chat_caps.get("supports_tools", False)
 
+            # Media marker for multimodal embeddings; the server randomizes this
+            # per startup unless LLAMA_MEDIA_MARKER is set, so we must read it
+            # from /props rather than hardcoding "<__media__>".
+            media_marker = props.get("media_marker")
+            if isinstance(media_marker, str) and media_marker:
+                self._media_marker = media_marker
+
             logger.info(
                 "llama.cpp model '%s' initialized — context: %s, vision: %s, audio: %s, tools: %s",
                 configured_model,
@@ -465,10 +474,11 @@ class LlamaCppClient(GenAIClient):
             jpeg_bytes = _to_jpeg(img)
             to_encode = jpeg_bytes if jpeg_bytes is not None else img
             encoded = base64.b64encode(to_encode).decode("utf-8")
-            # prompt_string must contain <__media__> placeholder for image tokenization
+            # prompt_string must contain the server's media marker placeholder.
+            # The marker is randomized per server startup (read from /props).
             content.append(
                 {
-                    "prompt_string": "<__media__>\n",
+                    "prompt_string": f"{self._media_marker}\n",
                     "multimodal_data": [encoded],  # type: ignore[dict-item]
                 }
             )

frigate/genai/openai.py

@@ -73,8 +73,17 @@ class OpenAIClient(GenAIClient):
                 **self.genai_config.runtime_options,
             }
             if response_format:
+                # OpenAI strict mode requires additionalProperties: false on the schema
+                if response_format.get("type") == "json_schema" and response_format.get(
+                    "json_schema", {}
+                ).get("strict"):
+                    schema = response_format.get("json_schema", {}).get("schema")
+                    if isinstance(schema, dict):
+                        schema["additionalProperties"] = False
                 request_params["response_format"] = response_format
+
             result = self.provider.chat.completions.create(**request_params)
+
             if (
                 result is not None
                 and hasattr(result, "choices")

frigate/video/ffmpeg.py

@@ -24,7 +24,7 @@ from frigate.config.camera.updater import (
 )
 from frigate.const import PROCESS_PRIORITY_HIGH
 from frigate.log import LogPipe
-from frigate.util.builtin import EventsPerSecond
+from frigate.util.builtin import EventsPerSecond, get_ffmpeg_arg_list
 from frigate.util.ffmpeg import start_or_restart_ffmpeg, stop_ffmpeg
 from frigate.util.image import (
     FrameManager,
@@ -34,6 +34,23 @@ from frigate.util.process import FrigateProcess
 
 logger = logging.getLogger(__name__)
 
+# all built-in record presets use this segment_time
+DEFAULT_RECORD_SEGMENT_TIME = 10
+
+
+def _get_record_segment_time(config: CameraConfig) -> int:
+    """Extract -segment_time from the camera's record output args."""
+    record_args = get_ffmpeg_arg_list(config.ffmpeg.output_args.record)
+
+    if record_args and record_args[0].startswith("preset"):
+        return DEFAULT_RECORD_SEGMENT_TIME
+
+    try:
+        idx = record_args.index("-segment_time")
+        return int(record_args[idx + 1])
+    except (ValueError, IndexError):
+        return DEFAULT_RECORD_SEGMENT_TIME
+
 
 def capture_frames(
     ffmpeg_process: sp.Popen[Any],
@@ -164,6 +181,12 @@ class CameraWatchdog(threading.Thread):
         self.latest_cache_segment_time: float = 0
         self.record_enable_time: datetime | None = None
 
+        # `valid` segments are published with the segment's start time, so the
+        # gap between consecutive publishes can reach 2 * segment_time. Pad the
+        # staleness threshold so it's never tighter than that worst case.
+        segment_time = _get_record_segment_time(self.config)
+        self.record_stale_threshold = max(120, 2 * segment_time + 30)
+
         # Stall tracking (based on last processed frame)
         self._stall_timestamps: deque[float] = deque()
         self._stall_active: bool = False
@@ -413,16 +436,17 @@ class CameraWatchdog(threading.Thread):
 
                     # ensure segments are still being created and that they have valid video data
                     # Skip checks during grace period to allow segments to start being created
+                    stale_window = timedelta(seconds=self.record_stale_threshold)
                     cache_stale = not in_grace_period and now_utc > (
-                        latest_cache_dt + timedelta(seconds=120)
+                        latest_cache_dt + stale_window
                     )
                     valid_stale = not in_grace_period and now_utc > (
-                        latest_valid_dt + timedelta(seconds=120)
+                        latest_valid_dt + stale_window
                     )
                     invalid_stale_condition = (
                         self.latest_invalid_segment_time > 0
                         and not in_grace_period
-                        and now_utc > (latest_invalid_dt + timedelta(seconds=120))
+                        and now_utc > (latest_invalid_dt + stale_window)
                         and self.latest_valid_segment_time
                         <= self.latest_invalid_segment_time
                     )
@@ -439,7 +463,7 @@ class CameraWatchdog(threading.Thread):
                             )
 
                         self.logger.error(
-                            f"{reason} for {self.config.name} in the last 120s. Restarting the ffmpeg record process..."
+                            f"{reason} for {self.config.name} in the last {self.record_stale_threshold}s. Restarting the ffmpeg record process..."
                         )
                         p["process"] = start_or_restart_ffmpeg(
                             p["cmd"],

frigate/watchdog.py

@@ -28,6 +28,7 @@ class MonitoredProcess:
     restart_timestamps: deque[float] = field(
         default_factory=lambda: deque(maxlen=MAX_RESTARTS)
     )
+    clean_exit_logged: bool = False
 
     def is_restarting_too_fast(self, now: float) -> bool:
         while (
@@ -72,7 +73,9 @@ class FrigateWatchdog(threading.Thread):
 
         exitcode = entry.process.exitcode
         if exitcode == 0:
-            logger.info("Process %s exited cleanly, not restarting", entry.name)
+            if not entry.clean_exit_logged:
+                logger.info("Process %s exited cleanly, not restarting", entry.name)
+                entry.clean_exit_logged = True
             return
 
         logger.warning(

web/src/components/overlay/MobileReviewSettingsDrawer.tsx

@@ -391,10 +391,8 @@ export default function MobileReviewSettingsDrawer({
             className="flex w-full items-center justify-center gap-2"
             aria-label={t("title", { ns: "views/replay" })}
             onClick={() => {
-              const now = new Date(latestTime * 1000);
-              now.setHours(now.getHours() - 1);
               setDebugReplayRange({
-                after: now.getTime() / 1000,
+                after: latestTime - 60,
                 before: latestTime,
               });
               setSelectedReplayOption("1");
@@ -541,11 +539,9 @@ export default function MobileReviewSettingsDrawer({
         return;
       }
 
-      const hours = parseInt(option);
+      const minutes = parseInt(option, 10);
       const end = latestTime;
-      const now = new Date(end * 1000);
-      now.setHours(now.getHours() - hours);
-      setDebugReplayRange({ after: now.getTime() / 1000, before: end });
+      setDebugReplayRange({ after: end - minutes * 60, before: end });
     };
 
     content = (

web/src/components/player/HlsVideoPlayer.tsx

@@ -396,7 +396,6 @@ export default function HlsVideoPlayer({
                 }}
               >
                 <ObjectTrackOverlay
-                  key={`overlay-${currentTime}`}
                   camera={camera}
                   showBoundingBoxes={!isPlaying}
                   currentTime={currentTime}

web/src/views/recording/RecordingView.tsx

@@ -728,10 +728,8 @@ export function RecordingView({
                   setShareTimestampOpen(true);
                 }}
                 onDebugReplayClick={() => {
-                  const now = new Date(timeRange.before * 1000);
-                  now.setHours(now.getHours() - 1);
                   setDebugReplayRange({
-                    after: now.getTime() / 1000,
+                    after: timeRange.before - 60,
                     before: timeRange.before,
                   });
                   setDebugReplayMode("select");

web/src/views/settings/MediaSyncSettingsView.tsx

@@ -10,13 +10,16 @@ import axios from "axios";
 import { toast } from "sonner";
 import { useJobStatus } from "@/api/ws";
 import { Switch } from "@/components/ui/switch";
-import { LuCheck, LuX } from "react-icons/lu";
+import { LuCheck, LuExternalLink, LuX } from "react-icons/lu";
 import { cn } from "@/lib/utils";
 import { formatUnixTimestampToDateTime } from "@/utils/dateUtil";
 import { MediaSyncResults, MediaSyncStats } from "@/types/ws";
+import { useDocDomain } from "@/hooks/use-doc-domain";
+import { Link } from "react-router-dom";
 
 export default function MediaSyncSettingsView() {
   const { t } = useTranslation("views/settings");
+  const { getLocaleDocUrl } = useDocDomain();
   const [selectedMediaTypes, setSelectedMediaTypes] = useState<string[]>([
     "all",
   ]);
@@ -109,13 +112,25 @@ export default function MediaSyncSettingsView() {
               <Heading as="h4" className="mb-2 hidden md:block">
                 {t("maintenance.sync.title")}
               </Heading>
-
               <div className="max-w-6xl">
                 <div className="mb-5 mt-2 flex max-w-5xl flex-col gap-2 text-sm text-muted-foreground">
                   <p>{t("maintenance.sync.desc")}</p>
+
+                  <div className="flex items-center text-primary-variant">
+                    <Link
+                      to={getLocaleDocUrl(
+                        "configuration/record#syncing-media-files-with-disk",
+                      )}
+                      target="_blank"
+                      rel="noopener noreferrer"
+                      className="inline"
+                    >
+                      {t("readTheDocumentation", { ns: "common" })}
+                      <LuExternalLink className="ml-2 inline-flex size-3" />
+                    </Link>
+                  </div>
                 </div>
               </div>
-
               <div className="space-y-6">
                 {/* Media Types Selection */}
                 <div>