diff --git a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run index 8f5b1c267..d3aa34236 100755 --- a/docker/main/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run +++ b/docker/main/rootfs/etc/s6-overlay/s6-rc.d/go2rtc/run @@ -55,7 +55,7 @@ function setup_homekit_config() { if [[ ! -f "${config_path}" ]]; then echo "[INFO] Creating empty HomeKit config file..." - echo '{}' > "${config_path}" + echo 'homekit: {}' > "${config_path}" fi # Convert YAML to JSON for jq processing @@ -70,12 +70,14 @@ function setup_homekit_config() { jq ' # Keep only the homekit section if it exists, otherwise empty object if has("homekit") then {homekit: .homekit} else {homekit: {}} end - ' "${temp_json}" > "${cleaned_json}" 2>/dev/null || echo '{"homekit": {}}' > "${cleaned_json}" + ' "${temp_json}" > "${cleaned_json}" 2>/dev/null || { + echo '{"homekit": {}}' > "${cleaned_json}" + } # Convert back to YAML and write to the config file yq eval -P "${cleaned_json}" > "${config_path}" 2>/dev/null || { echo "[WARNING] Failed to convert cleaned config to YAML, creating minimal config" - echo '{"homekit": {}}' > "${config_path}" + echo 'homekit: {}' > "${config_path}" } # Clean up temp files diff --git a/docs/docs/configuration/genai/review_summaries.md b/docs/docs/configuration/genai/review_summaries.md index 8a492f4b8..04a5b5e94 100644 --- a/docs/docs/configuration/genai/review_summaries.md +++ b/docs/docs/configuration/genai/review_summaries.md @@ -16,12 +16,13 @@ Review summaries provide structured JSON responses that are saved for each revie ``` - `title` (string): A concise, direct title that describes the purpose or overall action (e.g., "Person taking out trash", "Joe walking dog"). - `scene` (string): A narrative description of what happens across the sequence from start to finish, including setting, detected objects, and their observable actions. +- `shortSummary` (string): A brief 2-sentence summary of the scene, suitable for notifications. This is a condensed version of the scene description. - `confidence` (float): 0-1 confidence in the analysis. Higher confidence when objects/actions are clearly visible and context is unambiguous. - `other_concerns` (list): List of user-defined concerns that may need additional investigation. - `potential_threat_level` (integer): 0, 1, or 2 as defined below. ``` -This will show in multiple places in the UI to give additional context about each activity, and allow viewing more details when extra attention is required. Frigate's built in notifications will also automatically show the title and description when the data is available. +This will show in multiple places in the UI to give additional context about each activity, and allow viewing more details when extra attention is required. Frigate's built in notifications will automatically show the title and `shortSummary` when the data is available, while the full `scene` description is available in the UI for detailed review. ### Defining Typical Activity diff --git a/docs/docs/troubleshooting/memory.md b/docs/docs/troubleshooting/memory.md index 338037c7a..c74729e5f 100644 --- a/docs/docs/troubleshooting/memory.md +++ b/docs/docs/troubleshooting/memory.md @@ -36,7 +36,6 @@ Frigate processes are named using a module-based naming scheme. Common module na - `frigate.output` - Output processing - `frigate.audio_manager` - Audio processing - `frigate.embeddings` - Embeddings processing -- `frigate.embeddings_manager` - Embeddings manager You can also specify the full process name (including camera-specific identifiers) if you want to profile a specific camera: diff --git a/frigate/comms/webpush.py b/frigate/comms/webpush.py index 32eeb40e8..62cc12c9a 100644 --- a/frigate/comms/webpush.py +++ b/frigate/comms/webpush.py @@ -388,7 +388,7 @@ class WebPushClient(Communicator): else: title = base_title - message = payload["after"]["data"]["metadata"]["scene"] + message = payload["after"]["data"]["metadata"]["shortSummary"] else: zone_names = payload["after"]["data"]["zones"] formatted_zone_names = [] diff --git a/frigate/config/config.py b/frigate/config/config.py index 6342c13bf..a9c54976e 100644 --- a/frigate/config/config.py +++ b/frigate/config/config.py @@ -28,6 +28,7 @@ from frigate.util.builtin import ( get_ffmpeg_arg_list, ) from frigate.util.config import ( + CURRENT_CONFIG_VERSION, StreamInfoRetriever, convert_area_to_pixels, find_config_file, @@ -76,11 +77,12 @@ logger = logging.getLogger(__name__) yaml = YAML() -DEFAULT_CONFIG = """ +DEFAULT_CONFIG = f""" mqtt: enabled: False -cameras: {} # No cameras defined, UI wizard should be used +cameras: {{}} # No cameras defined, UI wizard should be used +version: {CURRENT_CONFIG_VERSION} """ DEFAULT_DETECTORS = {"cpu": {"type": "cpu"}} @@ -753,8 +755,7 @@ class FrigateConfig(FrigateBaseModel): if new_config and f.tell() == 0: f.write(DEFAULT_CONFIG) logger.info( - "Created default config file, see the getting started docs \ - for configuration https://docs.frigate.video/guides/getting_started" + "Created default config file, see the getting started docs for configuration: https://docs.frigate.video/guides/getting_started" ) f.seek(0) diff --git a/frigate/data_processing/post/object_descriptions.py b/frigate/data_processing/post/object_descriptions.py index 7bd38bfa8..cdb5f4fc3 100644 --- a/frigate/data_processing/post/object_descriptions.py +++ b/frigate/data_processing/post/object_descriptions.py @@ -86,7 +86,11 @@ class ObjectDescriptionProcessor(PostProcessorApi): and data["id"] not in self.early_request_sent ): if data["has_clip"] and data["has_snapshot"]: - event: Event = Event.get(Event.id == data["id"]) + try: + event: Event = Event.get(Event.id == data["id"]) + except DoesNotExist: + logger.error(f"Event {data['id']} not found") + return if ( not camera_config.objects.genai.objects diff --git a/frigate/data_processing/post/review_descriptions.py b/frigate/data_processing/post/review_descriptions.py index 0b12aa1a0..0a2754468 100644 --- a/frigate/data_processing/post/review_descriptions.py +++ b/frigate/data_processing/post/review_descriptions.py @@ -92,7 +92,7 @@ class ReviewDescriptionProcessor(PostProcessorApi): pixels_per_image = width * height tokens_per_image = pixels_per_image / 1250 - prompt_tokens = 3500 + prompt_tokens = 3800 response_tokens = 300 available_tokens = context_size - prompt_tokens - response_tokens max_frames = int(available_tokens / tokens_per_image) diff --git a/frigate/data_processing/post/types.py b/frigate/data_processing/post/types.py index 70fec9b34..44bb09fb0 100644 --- a/frigate/data_processing/post/types.py +++ b/frigate/data_processing/post/types.py @@ -8,6 +8,9 @@ class ReviewMetadata(BaseModel): scene: str = Field( description="A comprehensive description of the setting and entities, including relevant context and plausible inferences if supported by visual evidence." ) + shortSummary: str = Field( + description="A brief 2-sentence summary of the scene, suitable for notifications. Should capture the key activity and context without full detail." + ) confidence: float = Field( description="A float between 0 and 1 representing your overall confidence in this analysis." ) diff --git a/frigate/detectors/detection_runners.py b/frigate/detectors/detection_runners.py index 56b49ec67..fcbb41e66 100644 --- a/frigate/detectors/detection_runners.py +++ b/frigate/detectors/detection_runners.py @@ -139,8 +139,31 @@ class ONNXModelRunner(BaseModelRunner): ModelTypeEnum.dfine.value, ] - def __init__(self, ort: ort.InferenceSession): + @staticmethod + def is_concurrent_model(model_type: str | None) -> bool: + """Check if model requires thread locking for concurrent inference. + + Some models (like JinaV2) share one runner between text and vision embeddings + called from different threads, requiring thread synchronization. + """ + if not model_type: + return False + + # Import here to avoid circular imports + from frigate.embeddings.types import EnrichmentModelTypeEnum + + return model_type == EnrichmentModelTypeEnum.jina_v2.value + + def __init__(self, ort: ort.InferenceSession, model_type: str | None = None): self.ort = ort + self.model_type = model_type + + # Thread lock to prevent concurrent inference (needed for JinaV2 which shares + # one runner between text and vision embeddings called from different threads) + if self.is_concurrent_model(model_type): + self._inference_lock = threading.Lock() + else: + self._inference_lock = None def get_input_names(self) -> list[str]: return [input.name for input in self.ort.get_inputs()] @@ -150,6 +173,10 @@ class ONNXModelRunner(BaseModelRunner): return self.ort.get_inputs()[0].shape[3] def run(self, input: dict[str, Any]) -> Any | None: + if self._inference_lock: + with self._inference_lock: + return self.ort.run(None, input) + return self.ort.run(None, input) @@ -576,5 +603,6 @@ def get_optimized_runner( ), providers=providers, provider_options=options, - ) + ), + model_type=model_type, ) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index c74bc2310..1a0950cbb 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -633,7 +633,7 @@ class EmbeddingMaintainer(threading.Thread): camera, frame_name, _, _, motion_boxes, _ = data - if not camera or len(motion_boxes) == 0: + if not camera or len(motion_boxes) == 0 or camera not in self.config.cameras: return camera_config = self.config.cameras[camera] diff --git a/frigate/genai/__init__.py b/frigate/genai/__init__.py index 5e1a74279..7f0192912 100644 --- a/frigate/genai/__init__.py +++ b/frigate/genai/__init__.py @@ -101,6 +101,7 @@ When forming your description: Your response MUST be a flat JSON object with: - `title` (string): A concise, direct title that describes the primary action or event in the sequence, not just what you literally see. Use spatial context when available to make titles more meaningful. When multiple objects/actions are present, prioritize whichever is most prominent or occurs first. Use names from "Objects in Scene" based on what you visually observe. If you see both a name and an unidentified object of the same type but visually observe only one person/object, use ONLY the name. Examples: "Joe walking dog", "Person taking out trash", "Vehicle arriving in driveway", "Joe accessing vehicle", "Person leaving porch for driveway". - `scene` (string): A narrative description of what happens across the sequence from start to finish, in chronological order. Start by describing how the sequence begins, then describe the progression of events. **Describe all significant movements and actions in the order they occur.** For example, if a vehicle arrives and then a person exits, describe both actions sequentially. **Only describe actions you can actually observe happening in the frames provided.** Do not infer or assume actions that aren't visible (e.g., if you see someone walking but never see them sit, don't say they sat down). Include setting, detected objects, and their observable actions. Avoid speculation or filling in assumed behaviors. Your description should align with and support the threat level you assign. +- `shortSummary` (string): A brief 2-sentence summary of the scene, suitable for notifications. Should capture the key activity and context without full detail. This should be a condensed version of the scene description above. - `confidence` (float): 0-1 confidence in your analysis. Higher confidence when objects/actions are clearly visible and context is unambiguous. Lower confidence when the sequence is unclear, objects are partially obscured, or context is ambiguous. - `potential_threat_level` (integer): 0, 1, or 2 as defined in "Normal Activity Patterns for This Property" above. Your threat level must be consistent with your scene description and the guidance above. {get_concern_prompt()} @@ -192,6 +193,8 @@ Input format: Each event is a JSON object with: - "title", "scene", "confidence", "potential_threat_level" (0-2), "other_concerns", "camera", "time", "start_time", "end_time" - "context": array of related events from other cameras that occurred during overlapping time periods +**Note: Use the "scene" field for event descriptions in the report. Ignore any "shortSummary" field if present.** + Report Structure - Use this EXACT format: # Security Summary - {time_range} diff --git a/frigate/output/output.py b/frigate/output/output.py index 674c02b78..a44415000 100644 --- a/frigate/output/output.py +++ b/frigate/output/output.py @@ -139,9 +139,11 @@ class OutputProcess(FrigateProcess): if CameraConfigUpdateEnum.add in updates: for camera in updates["add"]: jsmpeg_cameras[camera] = JsmpegCamera( - cam_config, self.stop_event, websocket_server + self.config.cameras[camera], self.stop_event, websocket_server + ) + preview_recorders[camera] = PreviewRecorder( + self.config.cameras[camera] ) - preview_recorders[camera] = PreviewRecorder(cam_config) preview_write_times[camera] = 0 if ( diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index e3ae19159..bd4368ebe 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -1604,7 +1604,8 @@ function ObjectDetailsTab({ {config?.cameras[search?.camera].audio_transcription.enabled && search?.label == "speech" && - search?.end_time && ( + search?.end_time && + search?.has_clip && (