diff --git a/docs/docs/configuration/genai/config.md b/docs/docs/configuration/genai/config.md index 922f24a7ff..f04b699b75 100644 --- a/docs/docs/configuration/genai/config.md +++ b/docs/docs/configuration/genai/config.md @@ -226,7 +226,7 @@ Frigate supports any provider that implements the OpenAI API standard. This incl :::tip -For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: +Frigate reads the model's context size from the server when it is reported (such as by vLLM or OpenRouter). For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: ```yaml genai: @@ -448,7 +448,7 @@ To use a different OpenAI-compatible API endpoint, set the `OPENAI_BASE_URL` env :::tip -For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: +Frigate reads the model's context size from the server when it is reported (such as by vLLM or OpenRouter). For OpenAI-compatible servers (such as llama.cpp) that don't expose the configured context size in the API response, you can manually specify the context size in `provider_options`: ```yaml {6,7} genai: diff --git a/frigate/api/chat.py b/frigate/api/chat.py index a813563c1b..3c3d647661 100644 --- a/frigate/api/chat.py +++ b/frigate/api/chat.py @@ -306,7 +306,7 @@ async def _execute_search_objects_semantic( visual_distances: dict[str, float] = {} description_distances: dict[str, float] = {} try: - rows = context.search_thumbnail(semantic_query) + rows = await asyncio.to_thread(context.search_thumbnail, semantic_query) visual_distances = {row[0]: row[1] for row in rows} except Exception: logger.exception( @@ -314,7 +314,7 @@ async def _execute_search_objects_semantic( ) try: - rows = context.search_description(semantic_query) + rows = await asyncio.to_thread(context.search_description, semantic_query) description_distances = {row[0]: row[1] for row in rows} except Exception: logger.exception( @@ -448,7 +448,7 @@ async def _execute_find_similar_objects( try: if similarity_mode in ("visual", "fused"): - rows = context.search_thumbnail(anchor) + rows = await asyncio.to_thread(context.search_thumbnail, anchor) visual_distances = {row[0]: row[1] for row in rows} if similarity_mode in ("semantic", "fused"): @@ -457,7 +457,7 @@ async def _execute_find_similar_objects( or anchor.sub_label or anchor.label ) - rows = context.search_description(query_text) + rows = await asyncio.to_thread(context.search_description, query_text) description_distances = {row[0]: row[1] for row in rows} except Exception: logger.exception("Similarity search failed") diff --git a/frigate/api/event.py b/frigate/api/event.py index 5f55f1c831..011c6b887b 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -1561,9 +1561,8 @@ async def set_description( if len(new_description) > 0: # If semantic search is enabled, update the index if request.app.frigate_config.semantic_search.enabled: - context.update_description( - event_id, - new_description, + await asyncio.to_thread( + context.update_description, event_id, new_description ) else: # embeddings are always cleaned up so they don't outlive their description diff --git a/frigate/api/review.py b/frigate/api/review.py index 25b9070a62..29d8dc5b19 100644 --- a/frigate/api/review.py +++ b/frigate/api/review.py @@ -1,5 +1,6 @@ """Review apis.""" +import asyncio import datetime import logging from functools import reduce @@ -797,7 +798,7 @@ async def regenerate_review_description(request: Request, review_id: str): ) context: EmbeddingsContext = request.app.embeddings - context.regenerate_review_description(review_id) + await asyncio.to_thread(context.regenerate_review_description, review_id) return JSONResponse( content={ diff --git a/frigate/comms/detections_updater.py b/frigate/comms/detections_updater.py index fbd0750820..b94ac9607d 100644 --- a/frigate/comms/detections_updater.py +++ b/frigate/comms/detections_updater.py @@ -3,6 +3,8 @@ from enum import Enum from typing import Any +from frigate.const import FAST_QUEUE_TIMEOUT + from .zmq_proxy import Publisher, Subscriber @@ -33,7 +35,7 @@ class DetectionSubscriber(Subscriber): super().__init__(topic) def check_for_update( - self, timeout: float | None = None + self, timeout: float | None = FAST_QUEUE_TIMEOUT ) -> tuple[str, Any] | tuple[None, None] | None: return super().check_for_update(timeout) diff --git a/frigate/genai/plugins/openai.py b/frigate/genai/plugins/openai.py index fe4b63a636..70ae6a8e28 100644 --- a/frigate/genai/plugins/openai.py +++ b/frigate/genai/plugins/openai.py @@ -39,6 +39,28 @@ def _stats_from_openai_usage(usage: Any) -> dict[str, Any] | None: return stats or None +def _context_size_from_model(model: Any) -> int | None: + """Read the context window from an OpenAI-compatible /models entry. + + vLLM reports ``max_model_len``. OpenRouter reports ``context_length`` as the + largest window any of its providers offers, and ``top_provider.context_length`` + for the provider it routes to first. The latter can be smaller, so it is + preferred to avoid sending more than the serving provider accepts. + """ + top_provider = getattr(model, "top_provider", None) + candidates = ( + getattr(model, "max_model_len", None), + top_provider.get("context_length") if isinstance(top_provider, dict) else None, + getattr(model, "context_length", None), + ) + + for value in candidates: + if isinstance(value, int) and value > 0: + return value + + return None + + @register_genai_provider(GenAIProviderEnum.openai) class OpenAIClient(GenAIClient): """Generative AI client for Frigate using OpenAI.""" @@ -225,8 +247,9 @@ class OpenAIClient(GenAIClient): models = self.provider.models.list() for model in models.data: if model.id == self.genai_config.model: - if hasattr(model, "max_model_len") and model.max_model_len: - self.context_size = model.max_model_len + context_size = _context_size_from_model(model) + if context_size: + self.context_size = context_size logger.debug( f"Retrieved context size {self.context_size} for model {self.genai_config.model}" ) diff --git a/frigate/review/maintainer.py b/frigate/review/maintainer.py index f7a665c6db..dc92ab0d54 100644 --- a/frigate/review/maintainer.py +++ b/frigate/review/maintainer.py @@ -1007,6 +1007,13 @@ class ReviewSegmentMaintainer(threading.Thread): ) ) elif topic == DetectionTypeEnum.api: + # the event's segment has already ended + if manual_info["state"] == ManualEventState.end: + self.indefinite_events[camera].pop( + manual_info["event_id"], None + ) + continue + severity = self.get_manual_event_severity( camera, manual_info["label"] ) diff --git a/frigate/test/http_api/base_http_test.py b/frigate/test/http_api/base_http_test.py index 1efa3acdb1..fabd76a7e4 100644 --- a/frigate/test/http_api/base_http_test.py +++ b/frigate/test/http_api/base_http_test.py @@ -1,7 +1,10 @@ import datetime import logging import os +import struct +import tempfile import unittest +import zoneinfo from unittest.mock import patch from fastapi import Request @@ -20,6 +23,13 @@ from frigate.models import Event, Recordings, ReviewSegment from frigate.review.types import SeverityEnum from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS, TEST_DB_PRAGMAS +# Minimal TZif file for a zone fixed at UTC+0 named "UTC". +UTC_TZIF = ( + struct.pack(">4s16x6l", b"TZif", 0, 0, 0, 0, 1, 4) + + struct.pack(">lBB", 0, 0, 0) + + b"UTC\0" +) + class AuthTestClient(TestClient): """TestClient that automatically adds auth headers to all requests.""" @@ -36,6 +46,23 @@ class AuthTestClient(TestClient): class BaseTestHttp(unittest.TestCase): + @classmethod + def setUpClass(cls): + super().setUpClass() + + # Docker mounts of the host's /etc/localtime can overwrite the system + # zoneinfo/Etc/UTC file with a local zone. Shadow "UTC" with a known + # good file so the API's ZoneInfo("UTC") is real UTC. + tz_dir = tempfile.TemporaryDirectory() + with open(os.path.join(tz_dir.name, "UTC"), "wb") as f: + f.write(UTC_TZIF) + + zoneinfo.reset_tzpath(to=[tz_dir.name, *zoneinfo.TZPATH]) + zoneinfo.ZoneInfo.clear_cache() + cls.addClassCleanup(tz_dir.cleanup) + cls.addClassCleanup(zoneinfo.ZoneInfo.clear_cache) + cls.addClassCleanup(zoneinfo.reset_tzpath) + def setUp(self, models): # setup clean database for each test run migrate_db = SqliteDatabase(TEST_DB, pragmas=TEST_DB_PRAGMAS) diff --git a/frigate/test/http_api/test_http_ffprobe.py b/frigate/test/http_api/test_http_ffprobe.py index 5038c07c85..4d537452df 100644 --- a/frigate/test/http_api/test_http_ffprobe.py +++ b/frigate/test/http_api/test_http_ffprobe.py @@ -1,11 +1,13 @@ -"""Tests for credential scrubbing in the ffprobe API.""" +"""Tests for the ffprobe API.""" +import unittest from types import SimpleNamespace from unittest.mock import patch from fastapi.testclient import TestClient from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp +from frigate.util.services import ffprobe_stream class TestHttpFfprobe(BaseTestHttp): @@ -35,3 +37,17 @@ class TestHttpFfprobe(BaseTestHttp): self.assertEqual(len(lines), 2) self.assertNotIn("hunter2", " ".join(lines)) self.assertEqual(lines[1], "rtsp://*:*@10.0.0.1:554/video: Connection refused") + + +class TestFfprobeStream(unittest.TestCase): + def test_requests_codec_type(self): + ffmpeg = SimpleNamespace(ffprobe_path="ffprobe") + done = SimpleNamespace(returncode=0, stdout=b"{}", stderr=b"") + + for detailed in (False, True): + with patch("frigate.util.services.sp.run", return_value=done) as run: + ffprobe_stream(ffmpeg, "rtsp://10.0.0.1/live", detailed=detailed) + + cmd = run.call_args.args[0] + entries = cmd[cmd.index("-show_entries") + 1] + self.assertIn("codec_type", entries.removeprefix("stream=").split(",")) diff --git a/frigate/test/http_api/test_http_review.py b/frigate/test/http_api/test_http_review.py index 8f24ba24d4..8afe242263 100644 --- a/frigate/test/http_api/test_http_review.py +++ b/frigate/test/http_api/test_http_review.py @@ -1,4 +1,4 @@ -from datetime import datetime, timedelta +from datetime import UTC, datetime, timedelta from fastapi import Request from peewee import DoesNotExist @@ -9,6 +9,11 @@ from frigate.review.types import SeverityEnum from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp +def utc_day(dt: datetime) -> str: + """Return the UTC calendar day the summary endpoint buckets dt into.""" + return dt.astimezone(UTC).strftime("%Y-%m-%d") + + class TestHttpReview(BaseTestHttp): def setUp(self): super().setUp([Event, Recordings, ReviewSegment, UserReviewStatus]) @@ -362,7 +367,7 @@ class TestHttpReview(BaseTestHttp): assert response_json["last24Hours"]["total_alert"] == 1 assert response_json["last24Hours"]["total_detection"] == 0 - today_formatted = datetime.today().strftime("%Y-%m-%d") + today_formatted = utc_day(datetime.now()) assert response_json[today_formatted]["total_alert"] == 1 assert response_json[today_formatted]["total_detection"] == 0 @@ -379,7 +384,7 @@ class TestHttpReview(BaseTestHttp): assert response.status_code == 200 response_json = response.json() # e.g. '2024-11-24' - today_formatted = datetime.today().strftime("%Y-%m-%d") + today_formatted = utc_day(datetime.now()) expected_response = { "last24Hours": { "reviewed_alert": 0, @@ -404,7 +409,7 @@ class TestHttpReview(BaseTestHttp): assert response.status_code == 200 response_json = response.json() # e.g. '2024-11-24' - today_formatted = datetime.today().strftime("%Y-%m-%d") + today_formatted = utc_day(datetime.now()) expected_response = { "last24Hours": { "reviewed_alert": 0, @@ -439,9 +444,9 @@ class TestHttpReview(BaseTestHttp): assert response.status_code == 200 response_json = response.json() # e.g. '2024-11-24' - today_formatted = now.strftime("%Y-%m-%d") + today_formatted = utc_day(now) # e.g. '2024-11-19' - five_days_ago_formatted = five_days_ago.strftime("%Y-%m-%d") + five_days_ago_formatted = utc_day(five_days_ago) expected_response = { "last24Hours": { "reviewed_alert": 0, @@ -491,9 +496,9 @@ class TestHttpReview(BaseTestHttp): assert response.status_code == 200 response_json = response.json() # e.g. '2024-11-24' - today_formatted = now.strftime("%Y-%m-%d") + today_formatted = utc_day(now) # e.g. '2024-11-19' - five_days_ago_formatted = five_days_ago.strftime("%Y-%m-%d") + five_days_ago_formatted = utc_day(five_days_ago) expected_response = { "last24Hours": { "reviewed_alert": 0, @@ -549,7 +554,7 @@ class TestHttpReview(BaseTestHttp): assert response.status_code == 200 response_json = response.json() # e.g. '2024-11-19' - five_days_ago_formatted = five_days_ago.strftime("%Y-%m-%d") + five_days_ago_formatted = utc_day(five_days_ago) expected_response = { "last24Hours": { "reviewed_alert": None, diff --git a/frigate/test/test_detections_updater.py b/frigate/test/test_detections_updater.py new file mode 100644 index 0000000000..4147b39fbb --- /dev/null +++ b/frigate/test/test_detections_updater.py @@ -0,0 +1,73 @@ +"""Tests for the detection subscriber's default receive timeout.""" + +import json +import os +import tempfile +import threading +import time +import unittest +from unittest.mock import patch + +import zmq + +from frigate.comms import zmq_proxy +from frigate.comms.detections_updater import DetectionSubscriber, DetectionTypeEnum + + +class TestDetectionSubscriberTimeout(unittest.TestCase): + def setUp(self) -> None: + tmp = tempfile.TemporaryDirectory() + self.addCleanup(tmp.cleanup) + self.address = f"ipc://{os.path.join(tmp.name, 'proxy_sub')}" + + def _subscriber(self) -> DetectionSubscriber: + with patch.object(zmq_proxy, "SOCKET_SUB", self.address): + return DetectionSubscriber(DetectionTypeEnum.video.value) + + def test_default_check_returns_when_nothing_is_published(self) -> None: + subscriber = self._subscriber() + results: list = [] + thread = threading.Thread( + target=lambda: results.append(subscriber.check_for_update()), + daemon=True, + ) + thread.start() + thread.join(timeout=5) + + # a blocked check still owns the socket, so only stop it after returning + self.assertFalse( + thread.is_alive(), "check_for_update blocked with nothing published" + ) + subscriber.stop() + self.assertEqual(results, [(None, None)]) + + def test_default_check_returns_queued_detection(self) -> None: + context = zmq.Context() + publisher = context.socket(zmq.PUB) + publisher.bind(self.address) + subscriber = self._subscriber() + self.addCleanup(context.destroy, linger=0) + self.addCleanup(subscriber.stop) + + # wait for the subscription to reach the publisher before relying on it + deadline = time.monotonic() + 5 + while True: + publisher.send_string(f"detection/video {json.dumps('ready')}") + + if subscriber.check_for_update(timeout=0.05) == ("video", "ready"): + break + + self.assertLess(time.monotonic(), deadline, "subscription never joined") + + # drop any late readiness messages so only the real one is queued + while subscriber.check_for_update(timeout=0.05) != (None, None): + pass + + publisher.send_string(f"detection/video {json.dumps(['front', 1.0])}") + time.sleep(0.1) + + self.assertEqual(subscriber.check_for_update(), ("video", ["front", 1.0])) + + +if __name__ == "__main__": + unittest.main() diff --git a/frigate/test/test_genai_providers.py b/frigate/test/test_genai_providers.py index 0af70c75fc..10cb80356c 100644 --- a/frigate/test/test_genai_providers.py +++ b/frigate/test/test_genai_providers.py @@ -327,6 +327,76 @@ class TestOpenAIProvider(unittest.TestCase): async_client.close.assert_awaited_once() +class TestOpenAIContextSize(unittest.TestCase): + """Context size lookup from an OpenAI-compatible /models endpoint.""" + + # Neither model name contains "gpt", so the name-based default is 8192 and + # any other value must have come from the /models response. + MODEL = "google/gemini-2.5-flash" + + def _client_with_models(self, *entries, provider_options=None): + client = _make_client( + "openai", + model=self.MODEL, + api_key="k", + base_url="https://openrouter.ai/api/v1", + provider_options=provider_options or {}, + ) + client.provider.models.list = MagicMock( + return_value=SimpleNamespace(data=list(entries)) + ) + return client + + def test_vllm_max_model_len(self): + client = self._client_with_models( + SimpleNamespace(id=self.MODEL, max_model_len=32768) + ) + self.assertEqual(client.get_context_size(), 32768) + + def test_openrouter_prefers_top_provider_context_length(self): + # The top-level value is the largest any provider offers; the routed + # provider's smaller window is the one requests must fit in. + client = self._client_with_models( + SimpleNamespace(id="other/model", context_length=4096), + SimpleNamespace( + id=self.MODEL, + context_length=1048576, + top_provider={"context_length": 262144}, + ), + ) + self.assertEqual(client.get_context_size(), 262144) + + def test_openrouter_falls_back_to_context_length(self): + client = self._client_with_models( + SimpleNamespace( + id=self.MODEL, + context_length=1048576, + top_provider={"context_length": None}, + ) + ) + self.assertEqual(client.get_context_size(), 1048576) + + def test_default_when_model_reports_no_context(self): + client = self._client_with_models(SimpleNamespace(id=self.MODEL)) + self.assertEqual(client.get_context_size(), 8192) + + def test_provider_options_override_api(self): + client = self._client_with_models( + SimpleNamespace(id=self.MODEL, context_length=1048576), + provider_options={"context_size": 16000}, + ) + self.assertEqual(client.get_context_size(), 16000) + client.provider.models.list.assert_not_called() + + def test_result_is_cached(self): + client = self._client_with_models( + SimpleNamespace(id=self.MODEL, context_length=1048576) + ) + client.get_context_size() + client.get_context_size() + client.provider.models.list.assert_called_once() + + # --------------------------------------------------------------------------- # Gemini # --------------------------------------------------------------------------- diff --git a/frigate/test/test_review_flow.py b/frigate/test/test_review_flow.py index ebf7b981b7..314775161a 100644 --- a/frigate/test/test_review_flow.py +++ b/frigate/test/test_review_flow.py @@ -862,20 +862,41 @@ class TestSplitDetectionLabels(ReviewFlowTestCase): class TestManualEvents(ReviewFlowTestCase): def manual_event( - self, event_id: str, frame_time: float, sub_label: str | None = None + self, + event_id: str, + frame_time: float, + sub_label: str | None = None, + state: ManualEventState = ManualEventState.complete, ) -> tuple[str, tuple[Any, ...]]: - """Build a completed manual event as published by the object processor.""" + """Build a manual event as published by the object processor.""" return ( DetectionTypeEnum.api.value, ( CAMERA, frame_time, { - "state": ManualEventState.complete, + "state": state, "label": "person", "sub_label": sub_label, "event_id": event_id, - "end_time": frame_time + 5, + "end_time": ( + frame_time + 5 if state == ManualEventState.complete else None + ), + }, + ), + ) + + def end_event(self, event_id: str, end_time: float) -> tuple[str, tuple[Any, ...]]: + """Build the end message the events API publishes, which has no label.""" + return ( + DetectionTypeEnum.api.value, + ( + CAMERA, + end_time, + { + "state": ManualEventState.end, + "event_id": event_id, + "end_time": end_time, }, ), ) @@ -900,6 +921,41 @@ class TestManualEvents(ReviewFlowTestCase): self.assertCountEqual(data["objects"], ["dog", "person-verified", "person"]) self.assertEqual(data["sub_labels"], ["Bob"]) + def test_ending_event_after_its_segment_closed_is_ignored(self) -> None: + self.feed_messages(self.manual_event("m1", 1, state=ManualEventState.start)) + self.maintainer.config_subscriber.check_for_updates.side_effect = [ + {"record": [CAMERA]} + ] + self.feed() + self.maintainer.config_subscriber.check_for_updates.side_effect = None + self.feed_messages(self.end_event("m1", 20), self.manual_event("m2", 30)) + + self.assertEqual(self.non_update_summary(), [("end", "alert")]) + segment = self.maintainer.active_review_segments[CAMERA] + self.assertEqual(segment.detections, {"m2": "person"}) + + def test_ending_event_after_its_segment_ended_early_clears_it(self) -> None: + # a completed event ends the indefinite event's segment early + self.feed_messages( + self.manual_event("m1", 1, state=ManualEventState.start), + self.manual_event("m2", 2), + ) + self.feed((8, []), (8 + self.alert_cutoff, [])) + self.assertIsNone(self.maintainer.active_review_segments.get(CAMERA)) + + self.feed_messages( + self.end_event("m1", 100), + self.manual_event("m3", 200, state=ManualEventState.start), + self.end_event("m3", 210), + ) + self.feed((210 + self.alert_cutoff + 1, [])) + + self.assertEqual(self.maintainer.indefinite_events[CAMERA], {}) + self.assertIsNone(self.maintainer.active_review_segments.get(CAMERA)) + self.assertEqual( + self.non_update_summary(), [("end", "alert"), ("end", "alert")] + ) + if __name__ == "__main__": unittest.main() diff --git a/frigate/util/services.py b/frigate/util/services.py index dcb0a5b289..03eb61782d 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -1365,7 +1365,7 @@ def ffprobe_stream(ffmpeg, path: str, detailed: bool = False) -> sp.CompletedPro clean_path = escape_special_characters(path) # Base entries that are always included - stream_entries = "codec_long_name,width,height,bit_rate,duration,display_aspect_ratio,avg_frame_rate" + stream_entries = "codec_type,codec_long_name,width,height,bit_rate,duration,display_aspect_ratio,avg_frame_rate" # Additional detailed entries if detailed: diff --git a/web/src/utils/streamIssues.ts b/web/src/utils/streamIssues.ts index f763cc9c6d..2d05849d76 100644 --- a/web/src/utils/streamIssues.ts +++ b/web/src/utils/streamIssues.ts @@ -57,18 +57,8 @@ export function ffprobeToTestResult( } const streams = entry.stdout?.streams ?? []; - const videoStream = streams.find( - (s) => - s.codec_type === "video" || - s.codec_name?.includes("h264") || - s.codec_name?.includes("h265"), - ); - const audioStream = streams.find( - (s) => - s.codec_type === "audio" || - s.codec_name?.includes("aac") || - s.codec_name?.includes("mp3"), - ); + const videoStream = streams.find((s) => s.codec_type === "video"); + const audioStream = streams.find((s) => s.codec_type === "audio"); const resolution = videoStream ? `${videoStream.width}x${videoStream.height}`