From 5264a18dfab3318300be76c632dd8c6b350fb549 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Sat, 15 Feb 2025 07:56:45 -0600 Subject: [PATCH 01/19] Small fixes and docs tweaks (#16595) * don't clear url params if we're creating a new object mask * use correct threshold var in debug log * docs tweak for mjpeg cameras --- docs/docs/configuration/camera_specific.md | 4 ++-- .../data_processing/real_time/license_plate_processor.py | 2 +- web/src/pages/Settings.tsx | 9 +++++++-- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/docs/configuration/camera_specific.md b/docs/docs/configuration/camera_specific.md index ab5e605d29..fb4ce57146 100644 --- a/docs/docs/configuration/camera_specific.md +++ b/docs/docs/configuration/camera_specific.md @@ -22,7 +22,7 @@ Note that mjpeg cameras require encoding the video into h264 for recording, and ```yaml go2rtc: streams: - mjpeg_cam: "ffmpeg:{your_mjpeg_stream_url}#video=h264#hardware" # <- use hardware acceleration to create an h264 stream usable for other components. + mjpeg_cam: "ffmpeg:http://your_mjpeg_stream_url#video=h264#hardware" # <- use hardware acceleration to create an h264 stream usable for other components. cameras: ... @@ -85,7 +85,7 @@ This camera is H.265 only. To be able to play clips on some devices (like MacOs cameras: annkec800: # <------ Name the camera ffmpeg: - apple_compatibility: true # <- Adds compatibility with MacOS and iPhone + apple_compatibility: true # <- Adds compatibility with MacOS and iPhone output_args: record: preset-record-generic-audio-aac diff --git a/frigate/data_processing/real_time/license_plate_processor.py b/frigate/data_processing/real_time/license_plate_processor.py index b0c4e24617..2d64e5cdba 100644 --- a/frigate/data_processing/real_time/license_plate_processor.py +++ b/frigate/data_processing/real_time/license_plate_processor.py @@ -1062,7 +1062,7 @@ class LicensePlateProcessor(RealTimeProcessorApi): # Check against minimum confidence threshold if avg_confidence < self.lpr_config.recognition_threshold: logger.debug( - f"Average confidence {avg_confidence} is less than threshold ({self.lpr_config.threshold})" + f"Average confidence {avg_confidence} is less than threshold ({self.lpr_config.recognition_threshold})" ) return diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index 0fcc0414eb..fbf31a00a7 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -38,6 +38,7 @@ import NotificationView from "@/views/settings/NotificationsSettingsView"; import SearchSettingsView from "@/views/settings/SearchSettingsView"; import UiSettingsView from "@/views/settings/UiSettingsView"; import { useSearchEffect } from "@/hooks/use-overlay-state"; +import { useSearchParams } from "react-router-dom"; const allSettingsViews = [ "UI settings", @@ -58,6 +59,8 @@ export default function Settings() { const { data: config } = useSWR("config"); + const [searchParams] = useSearchParams(); + // available settings views const settingsViews = useMemo(() => { @@ -124,7 +127,8 @@ export default function Settings() { if (allSettingsViews.includes(page as SettingsType)) { setPage(page as SettingsType); } - return false; + // don't clear url params if we're creating a new object mask + return !searchParams.has("object_mask"); }); useSearchEffect("camera", (camera: string) => { @@ -132,7 +136,8 @@ export default function Settings() { if (cameraNames.includes(camera)) { setSelectedCamera(camera); } - return false; + // don't clear url params if we're creating a new object mask + return !searchParams.has("object_mask"); }); useEffect(() => { From 92553fa666d856f7f873068d93674f3bf89da314 Mon Sep 17 00:00:00 2001 From: Lukas Wolfsteiner Date: Sun, 16 Feb 2025 02:57:59 +0100 Subject: [PATCH 02/19] fix: Missing link to go2rtc' logging doc (#16606) Updated the link to go2rtc docs for logging configuration. --- docs/docs/configuration/advanced.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/configuration/advanced.md b/docs/docs/configuration/advanced.md index 3068ec8f84..c1f12ee089 100644 --- a/docs/docs/configuration/advanced.md +++ b/docs/docs/configuration/advanced.md @@ -32,7 +32,7 @@ Examples of available modules are: #### Go2RTC Logging -See [the go2rtc docs](for logging configuration) +See [the go2rtc docs](https://github.com/AlexxIT/go2rtc?tab=readme-ov-file#module-log) for logging configuration ```yaml go2rtc: From 349abc8d1b36f25a2a6cba418a1ee161aea9f9b6 Mon Sep 17 00:00:00 2001 From: Felipe Santos Date: Mon, 17 Feb 2025 01:24:13 -0300 Subject: [PATCH 03/19] Remove internal docker container IP from WebRTC candidates (#16618) --- .../rootfs/usr/local/go2rtc/create_config.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/docker/main/rootfs/usr/local/go2rtc/create_config.py b/docker/main/rootfs/usr/local/go2rtc/create_config.py index 1038af64cd..0e4aa7bd1c 100644 --- a/docker/main/rootfs/usr/local/go2rtc/create_config.py +++ b/docker/main/rootfs/usr/local/go2rtc/create_config.py @@ -66,29 +66,32 @@ elif go2rtc_config["log"].get("format") is None: go2rtc_config["log"]["format"] = "text" # ensure there is a default webrtc config -if not go2rtc_config.get("webrtc"): +if go2rtc_config.get("webrtc") is None: go2rtc_config["webrtc"] = {} # go2rtc should listen on 8555 tcp & udp by default -if not go2rtc_config["webrtc"].get("listen"): +if go2rtc_config["webrtc"].get("listen") is None: go2rtc_config["webrtc"]["listen"] = ":8555" -if not go2rtc_config["webrtc"].get("candidates", []): +if go2rtc_config["webrtc"].get("candidates") is None: default_candidates = [] # use internal candidate if it was discovered when running through the add-on - internal_candidate = os.environ.get( - "FRIGATE_GO2RTC_WEBRTC_CANDIDATE_INTERNAL", None - ) + internal_candidate = os.environ.get("FRIGATE_GO2RTC_WEBRTC_CANDIDATE_INTERNAL") if internal_candidate is not None: default_candidates.append(internal_candidate) # should set default stun server so webrtc can work default_candidates.append("stun:8555") - go2rtc_config["webrtc"] = {"candidates": default_candidates} -else: - print( - "[INFO] Not injecting WebRTC candidates into go2rtc config as it has been set manually", - ) + go2rtc_config["webrtc"]["candidates"] = default_candidates + +# This prevents WebRTC from attempting to establish a connection to the internal +# docker IPs which are not accessible from outside the container itself and just +# wastes time during negotiation. Note that this is only necessary because +# Frigate container doesn't run in host network mode. +if go2rtc_config["webrtc"].get("filter") is None: + go2rtc_config["webrtc"]["filter"] = {"candidates": []} +elif go2rtc_config["webrtc"]["filter"].get("candidates") is None: + go2rtc_config["webrtc"]["filter"]["candidates"] = [] # sets default RTSP response to be equivalent to ?video=h264,h265&audio=aac # this means user does not need to specify audio codec when using restream From 2020cdffd5f3fcd006ec27e3ce9ec215f15a260a Mon Sep 17 00:00:00 2001 From: Mitch Ross Date: Mon, 17 Feb 2025 08:17:15 -0500 Subject: [PATCH 04/19] Fix prometheus client exporter (#16620) * wip * wip * put it back * formatter * Delete hailort.log * Delete hailort.log * lint --------- Co-authored-by: Nicolas Mowen --- docker-compose.yml | 2 +- docker/main/requirements-wheels.txt | 1 + frigate/api/app.py | 12 +- frigate/stats/prometheus.py | 664 ++++++++++++++++++++-------- 4 files changed, 486 insertions(+), 193 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f368805939..2d905d385c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,4 +38,4 @@ services: container_name: mqtt image: eclipse-mosquitto:1.6 ports: - - "1883:1883" + - "1883:1883" \ No newline at end of file diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index bb4ac622ba..e43e741558 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -68,3 +68,4 @@ netaddr==0.8.* netifaces==0.10.* verboselogs==1.7.* virtualenv==20.17.* +prometheus-client == 0.21.* \ No newline at end of file diff --git a/frigate/api/app.py b/frigate/api/app.py index 52e686af19..c55e36a4b9 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -20,7 +20,6 @@ from fastapi.params import Depends from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse from markupsafe import escape from peewee import operator -from prometheus_client import CONTENT_TYPE_LATEST, generate_latest from pydantic import ValidationError from frigate.api.defs.query.app_query_parameters import AppTimelineHourlyQueryParameters @@ -28,6 +27,7 @@ from frigate.api.defs.request.app_body import AppConfigSetBody from frigate.api.defs.tags import Tags from frigate.config import FrigateConfig from frigate.models import Event, Timeline +from frigate.stats.prometheus import get_metrics, update_metrics from frigate.util.builtin import ( clean_camera_user_pass, get_tz_modifiers, @@ -113,9 +113,13 @@ def stats_history(request: Request, keys: str = None): @router.get("/metrics") -def metrics(): - """Expose Prometheus metrics endpoint""" - return Response(content=generate_latest(), media_type=CONTENT_TYPE_LATEST) +def metrics(request: Request): + """Expose Prometheus metrics endpoint and update metrics with latest stats""" + # Retrieve the latest statistics and update the Prometheus metrics + stats = request.app.stats_emitter.get_latest_stats() + update_metrics(stats) + content, content_type = get_metrics() + return Response(content=content, media_type=content_type) @router.get("/config") diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index a43c091e27..015e551afe 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -1,207 +1,495 @@ -from typing import Dict +import logging +import re -from prometheus_client import ( - CONTENT_TYPE_LATEST, - Counter, - Gauge, - Info, - generate_latest, -) - -# System metrics -SYSTEM_INFO = Info("frigate_system", "System information") -CPU_USAGE = Gauge( - "frigate_cpu_usage_percent", - "Process CPU usage %", - ["pid", "name", "process", "type", "cmdline"], -) -MEMORY_USAGE = Gauge( - "frigate_mem_usage_percent", - "Process memory usage %", - ["pid", "name", "process", "type", "cmdline"], -) - -# Camera metrics -CAMERA_FPS = Gauge( - "frigate_camera_fps", - "Frames per second being consumed from your camera", - ["camera_name"], -) -DETECTION_FPS = Gauge( - "frigate_detection_fps", - "Number of times detection is run per second", - ["camera_name"], -) -PROCESS_FPS = Gauge( - "frigate_process_fps", - "Frames per second being processed by frigate", - ["camera_name"], -) -SKIPPED_FPS = Gauge( - "frigate_skipped_fps", "Frames per second skipped for processing", ["camera_name"] -) -DETECTION_ENABLED = Gauge( - "frigate_detection_enabled", "Detection enabled for camera", ["camera_name"] -) -AUDIO_DBFS = Gauge("frigate_audio_dBFS", "Audio dBFS for camera", ["camera_name"]) -AUDIO_RMS = Gauge("frigate_audio_rms", "Audio RMS for camera", ["camera_name"]) - -# Detector metrics -DETECTOR_INFERENCE = Gauge( - "frigate_detector_inference_speed_seconds", - "Time spent running object detection in seconds", - ["name"], -) -DETECTOR_START = Gauge( - "frigate_detection_start", "Detector start time (unix timestamp)", ["name"] -) - -# GPU metrics -GPU_USAGE = Gauge("frigate_gpu_usage_percent", "GPU utilisation %", ["gpu_name"]) -GPU_MEMORY = Gauge("frigate_gpu_mem_usage_percent", "GPU memory usage %", ["gpu_name"]) - -# Storage metrics -STORAGE_FREE = Gauge("frigate_storage_free_bytes", "Storage free bytes", ["storage"]) -STORAGE_TOTAL = Gauge("frigate_storage_total_bytes", "Storage total bytes", ["storage"]) -STORAGE_USED = Gauge("frigate_storage_used_bytes", "Storage used bytes", ["storage"]) -STORAGE_MOUNT = Info( - "frigate_storage_mount_type", "Storage mount type", ["mount_type", "storage"] -) - -# Service metrics -UPTIME = Gauge("frigate_service_uptime_seconds", "Uptime seconds") -LAST_UPDATE = Gauge( - "frigate_service_last_updated_timestamp", "Stats recorded time (unix timestamp)" -) -TEMPERATURE = Gauge("frigate_device_temperature", "Device Temperature", ["device"]) - -# Event metrics -CAMERA_EVENTS = Counter( - "frigate_camera_events", - "Count of camera events since exporter started", - ["camera", "label"], +from prometheus_client import CONTENT_TYPE_LATEST, generate_latest +from prometheus_client.core import ( + REGISTRY, + CounterMetricFamily, + GaugeMetricFamily, + InfoMetricFamily, ) -def update_metrics(stats: Dict) -> None: - """Update Prometheus metrics based on Frigate stats""" - try: - # Update process metrics - if "cpu_usages" in stats: - for pid, proc_stats in stats["cpu_usages"].items(): - cmdline = proc_stats.get("cmdline", "") - process_type = "Other" - process_name = cmdline +class CustomCollector(object): + def __init__(self, _url): + self.process_stats = {} + self.previous_event_id = None + self.previous_event_start_time = None + self.all_events = {} - CPU_USAGE.labels( - pid=pid, - name=process_name, - process=process_name, - type=process_type, - cmdline=cmdline, - ).set(float(proc_stats["cpu"])) + def add_metric(self, metric, label, stats, key, multiplier=1.0): # Now a method + try: + string = str(stats[key]) + value = float(re.findall(r"-?\d*\.?\d*", string)[0]) + metric.add_metric(label, value * multiplier) + except (KeyError, TypeError, IndexError, ValueError): + pass - MEMORY_USAGE.labels( - pid=pid, - name=process_name, - process=process_name, - type=process_type, - cmdline=cmdline, - ).set(float(proc_stats["mem"])) + def add_metric_process( + self, + metric, + camera_stats, + camera_name, + pid_name, + process_name, + cpu_or_memory, + process_type, + ): + try: + pid = str(camera_stats[pid_name]) + label_values = [pid, camera_name, process_name, process_type] + try: + # new frigate:0.13.0-beta3 stat 'cmdline' + label_values.append(self.process_stats[pid]["cmdline"]) + except KeyError: + pass + metric.add_metric(label_values, self.process_stats[pid][cpu_or_memory]) + del self.process_stats[pid][cpu_or_memory] + except (KeyError, TypeError, IndexError): + pass - # Update camera metrics - if "cameras" in stats: - for camera_name, camera_stats in stats["cameras"].items(): - if "camera_fps" in camera_stats: - CAMERA_FPS.labels(camera_name=camera_name).set( - camera_stats["camera_fps"] - ) - if "detection_fps" in camera_stats: - DETECTION_FPS.labels(camera_name=camera_name).set( - camera_stats["detection_fps"] - ) - if "process_fps" in camera_stats: - PROCESS_FPS.labels(camera_name=camera_name).set( - camera_stats["process_fps"] - ) - if "skipped_fps" in camera_stats: - SKIPPED_FPS.labels(camera_name=camera_name).set( - camera_stats["skipped_fps"] - ) - if "detection_enabled" in camera_stats: - DETECTION_ENABLED.labels(camera_name=camera_name).set( - camera_stats["detection_enabled"] - ) - if "audio_dBFS" in camera_stats: - AUDIO_DBFS.labels(camera_name=camera_name).set( - camera_stats["audio_dBFS"] - ) - if "audio_rms" in camera_stats: - AUDIO_RMS.labels(camera_name=camera_name).set( - camera_stats["audio_rms"] - ) + def collect(self): + stats = self.process_stats # Assign self.process_stats to local variable stats - # Update detector metrics - if "detectors" in stats: - for name, detector in stats["detectors"].items(): - if "inference_speed" in detector: - DETECTOR_INFERENCE.labels(name=name).set( - detector["inference_speed"] * 0.001 - ) # ms to seconds - if "detection_start" in detector: - DETECTOR_START.labels(name=name).set(detector["detection_start"]) + try: + self.process_stats = stats["cpu_usages"] + except KeyError: + pass - # Update GPU metrics - if "gpu_usages" in stats: - for gpu_name, gpu_stats in stats["gpu_usages"].items(): - if "gpu" in gpu_stats: - GPU_USAGE.labels(gpu_name=gpu_name).set(float(gpu_stats["gpu"])) - if "mem" in gpu_stats: - GPU_MEMORY.labels(gpu_name=gpu_name).set(float(gpu_stats["mem"])) + # process stats for cameras, detectors and other + cpu_usages = GaugeMetricFamily( + "frigate_cpu_usage_percent", + "Process CPU usage %", + labels=["pid", "name", "process", "type", "cmdline"], + ) + mem_usages = GaugeMetricFamily( + "frigate_mem_usage_percent", + "Process memory usage %", + labels=["pid", "name", "process", "type", "cmdline"], + ) - # Update service metrics - if "service" in stats: - service = stats["service"] + # camera stats + audio_dBFS = GaugeMetricFamily( + "frigate_audio_dBFS", "Audio dBFS for camera", labels=["camera_name"] + ) + audio_rms = GaugeMetricFamily( + "frigate_audio_rms", "Audio RMS for camera", labels=["camera_name"] + ) + camera_fps = GaugeMetricFamily( + "frigate_camera_fps", + "Frames per second being consumed from your camera.", + labels=["camera_name"], + ) + detection_enabled = GaugeMetricFamily( + "frigate_detection_enabled", + "Detection enabled for camera", + labels=["camera_name"], + ) + detection_fps = GaugeMetricFamily( + "frigate_detection_fps", + "Number of times detection is run per second.", + labels=["camera_name"], + ) + process_fps = GaugeMetricFamily( + "frigate_process_fps", + "Frames per second being processed by frigate.", + labels=["camera_name"], + ) + skipped_fps = GaugeMetricFamily( + "frigate_skipped_fps", + "Frames per second skip for processing by frigate.", + labels=["camera_name"], + ) - if "uptime" in service: - UPTIME.set(service["uptime"]) - if "last_updated" in service: - LAST_UPDATE.set(service["last_updated"]) + # read camera stats assuming version < frigate:0.13.0-beta3 + cameras = stats + try: + # try to read camera stats in case >= frigate:0.13.0-beta3 + cameras = stats["cameras"] + except KeyError: + pass - # Storage metrics - if "storage" in service: - for path, storage in service["storage"].items(): - if "free" in storage: - STORAGE_FREE.labels(storage=path).set( - storage["free"] * 1e6 - ) # MB to bytes - if "total" in storage: - STORAGE_TOTAL.labels(storage=path).set(storage["total"] * 1e6) - if "used" in storage: - STORAGE_USED.labels(storage=path).set(storage["used"] * 1e6) - if "mount_type" in storage: - STORAGE_MOUNT.labels(storage=path).info( - {"mount_type": storage["mount_type"], "storage": path} - ) + for camera_name, camera_stats in cameras.items(): + self.add_metric(audio_dBFS, [camera_name], camera_stats, "audio_dBFS") + self.add_metric(audio_rms, [camera_name], camera_stats, "audio_rms") + self.add_metric(camera_fps, [camera_name], camera_stats, "camera_fps") + self.add_metric( + detection_enabled, [camera_name], camera_stats, "detection_enabled" + ) + self.add_metric(detection_fps, [camera_name], camera_stats, "detection_fps") + self.add_metric(process_fps, [camera_name], camera_stats, "process_fps") + self.add_metric(skipped_fps, [camera_name], camera_stats, "skipped_fps") - # Temperature metrics - if "temperatures" in service: - for device, temp in service["temperatures"].items(): - TEMPERATURE.labels(device=device).set(temp) + self.add_metric_process( + cpu_usages, + camera_stats, + camera_name, + "ffmpeg_pid", + "ffmpeg", + "cpu", + "Camera", + ) + self.add_metric_process( + cpu_usages, + camera_stats, + camera_name, + "capture_pid", + "capture", + "cpu", + "Camera", + ) + self.add_metric_process( + cpu_usages, camera_stats, camera_name, "pid", "detect", "cpu", "Camera" + ) - # Version info - if "version" in service and "latest_version" in service: - SYSTEM_INFO.info( - { - "version": service["version"], - "latest_version": service["latest_version"], - } + self.add_metric_process( + mem_usages, + camera_stats, + camera_name, + "ffmpeg_pid", + "ffmpeg", + "mem", + "Camera", + ) + self.add_metric_process( + mem_usages, + camera_stats, + camera_name, + "capture_pid", + "capture", + "mem", + "Camera", + ) + self.add_metric_process( + mem_usages, camera_stats, camera_name, "pid", "detect", "mem", "Camera" + ) + + yield audio_dBFS + yield audio_rms + yield camera_fps + yield detection_enabled + yield detection_fps + yield process_fps + yield skipped_fps + + # bandwidth stats + bandwidth_usages = GaugeMetricFamily( + "frigate_bandwidth_usages_kBps", + "bandwidth usages kilobytes per second", + labels=["pid", "name", "process", "cmdline"], + ) + + try: + for b_pid, b_stats in stats["bandwidth_usages"].items(): + label = [b_pid] # pid label + try: + n = stats["cpu_usages"][b_pid]["cmdline"] + for p_name, p_stats in stats["processes"].items(): + if str(p_stats["pid"]) == b_pid: + n = p_name + break + + # new frigate:0.13.0-beta3 stat 'cmdline' + label.append(n) # name label + label.append(stats["cpu_usages"][b_pid]["cmdline"]) # process label + label.append(stats["cpu_usages"][b_pid]["cmdline"]) # cmdline label + self.add_metric(bandwidth_usages, label, b_stats, "bandwidth") + except KeyError: + pass + except KeyError: + pass + + yield bandwidth_usages + + # detector stats + try: + yield GaugeMetricFamily( + "frigate_detection_total_fps", + "Sum of detection_fps across all cameras and detectors.", + value=stats["detection_fps"], + ) + except KeyError: + pass + + detector_inference_speed = GaugeMetricFamily( + "frigate_detector_inference_speed_seconds", + "Time spent running object detection in seconds.", + labels=["name"], + ) + + detector_detection_start = GaugeMetricFamily( + "frigate_detection_start", + "Detector start time (unix timestamp)", + labels=["name"], + ) + + try: + for detector_name, detector_stats in stats["detectors"].items(): + self.add_metric( + detector_inference_speed, + [detector_name], + detector_stats, + "inference_speed", + 0.001, + ) # ms to seconds + self.add_metric( + detector_detection_start, + [detector_name], + detector_stats, + "detection_start", ) + self.add_metric_process( + cpu_usages, + stats["detectors"], + detector_name, + "pid", + "detect", + "cpu", + "Detector", + ) + self.add_metric_process( + mem_usages, + stats["detectors"], + detector_name, + "pid", + "detect", + "mem", + "Detector", + ) + except KeyError: + pass + yield detector_inference_speed + yield detector_detection_start + + # detector process stats + try: + for detector_name, detector_stats in stats["detectors"].items(): + p_pid = str(detector_stats["pid"]) + label = [p_pid] # pid label + try: + # new frigate:0.13.0-beta3 stat 'cmdline' + label.append(detector_name) # name label + label.append(detector_name) # process label + label.append("detectors") # type label + label.append(self.process_stats[p_pid]["cmdline"]) # cmdline label + self.add_metric(cpu_usages, label, self.process_stats[p_pid], "cpu") + self.add_metric(mem_usages, label, self.process_stats[p_pid], "mem") + del self.process_stats[p_pid] + except KeyError: + pass + + except KeyError: + pass + + # other named process stats + try: + for process_name, process_stats in stats["processes"].items(): + p_pid = str(process_stats["pid"]) + label = [p_pid] # pid label + try: + # new frigate:0.13.0-beta3 stat 'cmdline' + label.append(process_name) # name label + label.append(process_name) # process label + label.append(process_name) # type label + label.append(self.process_stats[p_pid]["cmdline"]) # cmdline label + self.add_metric(cpu_usages, label, self.process_stats[p_pid], "cpu") + self.add_metric(mem_usages, label, self.process_stats[p_pid], "mem") + del self.process_stats[p_pid] + except KeyError: + pass + + except KeyError: + pass + + # remaining process stats + try: + for process_id, pid_stats in self.process_stats.items(): + label = [process_id] # pid label + try: + # new frigate:0.13.0-beta3 stat 'cmdline' + label.append(pid_stats["cmdline"]) # name label + label.append(pid_stats["cmdline"]) # process label + label.append("Other") # type label + label.append(pid_stats["cmdline"]) # cmdline label + except KeyError: + pass + self.add_metric(cpu_usages, label, pid_stats, "cpu") + self.add_metric(mem_usages, label, pid_stats, "mem") + except KeyError: + pass + + yield cpu_usages + yield mem_usages + + # gpu stats + gpu_usages = GaugeMetricFamily( + "frigate_gpu_usage_percent", "GPU utilisation %", labels=["gpu_name"] + ) + gpu_mem_usages = GaugeMetricFamily( + "frigate_gpu_mem_usage_percent", "GPU memory usage %", labels=["gpu_name"] + ) + + try: + for gpu_name, gpu_stats in stats["gpu_usages"].items(): + self.add_metric(gpu_usages, [gpu_name], gpu_stats, "gpu") + self.add_metric(gpu_mem_usages, [gpu_name], gpu_stats, "mem") + except KeyError: + pass + + yield gpu_usages + yield gpu_mem_usages + + # service stats + uptime_seconds = GaugeMetricFamily( + "frigate_service_uptime_seconds", "Uptime seconds" + ) + last_updated_timestamp = GaugeMetricFamily( + "frigate_service_last_updated_timestamp", + "Stats recorded time (unix timestamp)", + ) + + try: + service_stats = stats["service"] + self.add_metric(uptime_seconds, [""], service_stats, "uptime") + self.add_metric(last_updated_timestamp, [""], service_stats, "last_updated") + + info = { + "latest_version": stats["service"]["latest_version"], + "version": stats["service"]["version"], + } + yield InfoMetricFamily( + "frigate_service", "Frigate version info", value=info + ) + + except KeyError: + pass + + yield uptime_seconds + yield last_updated_timestamp + + temperatures = GaugeMetricFamily( + "frigate_device_temperature", "Device Temperature", labels=["device"] + ) + try: + for device_name in stats["service"]["temperatures"]: + self.add_metric( + temperatures, + [device_name], + stats["service"]["temperatures"], + device_name, + ) + except KeyError: + pass + + yield temperatures + + storage_free = GaugeMetricFamily( + "frigate_storage_free_bytes", "Storage free bytes", labels=["storage"] + ) + storage_mount_type = InfoMetricFamily( + "frigate_storage_mount_type", + "Storage mount type", + labels=["mount_type", "storage"], + ) + storage_total = GaugeMetricFamily( + "frigate_storage_total_bytes", "Storage total bytes", labels=["storage"] + ) + storage_used = GaugeMetricFamily( + "frigate_storage_used_bytes", "Storage used bytes", labels=["storage"] + ) + + try: + for storage_path, storage_stats in stats["service"]["storage"].items(): + self.add_metric( + storage_free, [storage_path], storage_stats, "free", 1e6 + ) # MB to bytes + self.add_metric( + storage_total, [storage_path], storage_stats, "total", 1e6 + ) # MB to bytes + self.add_metric( + storage_used, [storage_path], storage_stats, "used", 1e6 + ) # MB to bytes + storage_mount_type.add_metric( + storage_path, + { + "mount_type": storage_stats["mount_type"], + "storage": storage_path, + }, + ) + except KeyError: + pass + + yield storage_free + yield storage_mount_type + yield storage_total + yield storage_used + + # count events + events = [] + + if len(events) > 0: + # events[0] is newest event, last element is oldest, don't need to sort + + if not self.previous_event_id: + # ignore all previous events on startup, prometheus might have already counted them + self.previous_event_id = events[0]["id"] + self.previous_event_start_time = int(events[0]["start_time"]) + + for event in events: + # break if event already counted + if event["id"] == self.previous_event_id: + break + + # break if event starts before previous event + if event["start_time"] < self.previous_event_start_time: + break + + # store counted events in a dict + try: + cam = self.all_events[event["camera"]] + try: + cam[event["label"]] += 1 + except KeyError: + # create label dict if not exists + cam.update({event["label"]: 1}) + except KeyError: + # create camera and label dict if not exists + self.all_events.update({event["camera"]: {event["label"]: 1}}) + + # don't recount events next time + self.previous_event_id = events[0]["id"] + self.previous_event_start_time = int(events[0]["start_time"]) + + camera_events = CounterMetricFamily( + "frigate_camera_events", + "Count of camera events since exporter started", + labels=["camera", "label"], + ) + + for camera, cam_dict in self.all_events.items(): + for label, label_value in cam_dict.items(): + camera_events.add_metric([camera, label], label_value) + + yield camera_events + + +collector = CustomCollector(None) +REGISTRY.register(collector) + + +def update_metrics(stats): + """Updates the Prometheus metrics with the given stats data.""" + try: + collector.process_stats = stats # Directly assign the stats data + # Important: Since we are not fetching from URL, we need to manually call collect + for _ in collector.collect(): + pass except Exception as e: - print(f"Error updating Prometheus metrics: {str(e)}") + logging.error(f"Error updating metrics: {e}") -def get_metrics() -> tuple[str, str]: - """Get Prometheus metrics in text format""" - return generate_latest(), CONTENT_TYPE_LATEST +def get_metrics(): + """Returns the Prometheus metrics in text format.""" + content = generate_latest(REGISTRY) # Use generate_latest + return content, CONTENT_TYPE_LATEST From 7b7387a68c7fdbe4bab268f67351b4f10bbc63cf Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 17 Feb 2025 06:38:29 -0700 Subject: [PATCH 05/19] Update info for face recognition (#16629) --- docs/docs/configuration/face_recognition.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/docs/configuration/face_recognition.md b/docs/docs/configuration/face_recognition.md index 22968762ab..aaab92e6de 100644 --- a/docs/docs/configuration/face_recognition.md +++ b/docs/docs/configuration/face_recognition.md @@ -5,11 +5,7 @@ title: Face Recognition Face recognition allows people to be assigned names and when their face is recognized Frigate will assign the person's name as a sub label. This information is included in the UI, filters, as well as in notifications. -Frigate has support for FaceNet to create face embeddings, which runs locally. Embeddings are then saved to Frigate's database. - -## Minimum System Requirements - -Face recognition works by running a large AI model locally on your system. Systems without a GPU will not run Face Recognition reliably or at all. +Frigate has support for CV2 Local Binary Pattern Face Recognizer to recognize faces, which runs locally. A lightweight face landmark detection model is also used to align faces before running them through the face recognizer. ## Configuration From 1e709f5b3ffcba4d017e95c8754140975087eb6c Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Mon, 17 Feb 2025 06:57:05 -0700 Subject: [PATCH 06/19] Update coral deps and remove unused pycoral (#16630) --- docker/main/install_deps.sh | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/docker/main/install_deps.sh b/docker/main/install_deps.sh index f8f68398f3..ee84f6a141 100755 --- a/docker/main/install_deps.sh +++ b/docker/main/install_deps.sh @@ -24,20 +24,18 @@ apt-get -qq install --no-install-recommends -y \ mkdir -p -m 600 /root/.gnupg # install coral runtime -wget -q -O /tmp/libedgetpu1-max.deb "https://github.com/feranick/libedgetpu/releases/download/16.0TF2.17.0-1/libedgetpu1-max_16.0tf2.17.0-1.bookworm_${TARGETARCH}.deb" +wget -q -O /tmp/libedgetpu1-max.deb "https://github.com/feranick/libedgetpu/releases/download/16.0TF2.17.1-1/libedgetpu1-max_16.0tf2.17.1-1.bookworm_${TARGETARCH}.deb" unset DEBIAN_FRONTEND yes | dpkg -i /tmp/libedgetpu1-max.deb && export DEBIAN_FRONTEND=noninteractive rm /tmp/libedgetpu1-max.deb # install python3 & tflite runtime if [[ "${TARGETARCH}" == "amd64" ]]; then - pip3 install --break-system-packages https://github.com/feranick/TFlite-builds/releases/download/v2.17.0/tflite_runtime-2.17.0-cp311-cp311-linux_x86_64.whl - pip3 install --break-system-packages https://github.com/feranick/pycoral/releases/download/2.0.2TF2.17.0/pycoral-2.0.2-cp311-cp311-linux_x86_64.whl + pip3 install --break-system-packages https://github.com/frigate-nvr/TFlite-builds/releases/download/v2.17.1/tflite_runtime-2.17.1-cp311-cp311-linux_x86_64.whl fi if [[ "${TARGETARCH}" == "arm64" ]]; then - pip3 install --break-system-packages https://github.com/feranick/TFlite-builds/releases/download/v2.17.0/tflite_runtime-2.17.0-cp311-cp311-linux_aarch64.whl - pip3 install --break-system-packages https://github.com/feranick/pycoral/releases/download/2.0.2TF2.17.0/pycoral-2.0.2-cp311-cp311-linux_aarch64.whl + pip3 install --break-system-packages https://github.com/feranick/TFlite-builds/releases/download/v2.17.1/tflite_runtime-2.17.1-cp311-cp311-linux_aarch64.whl fi # btbn-ffmpeg -> amd64 From 3f07d2d37ce8883d97a431c9fe5b91207a641049 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 17 Feb 2025 08:19:03 -0600 Subject: [PATCH 07/19] Improve notifications (#16632) * add notification cooldown * cooldown docs * show alert box when notifications are used in an insecure context * add ability to suspend notifications from dashboard context menu --- .cspell/frigate-dictionary.txt | 1 + docs/docs/configuration/notifications.md | 29 +++- docs/docs/configuration/reference.md | 2 + frigate/comms/webpush.py | 31 +++- frigate/config/camera/notification.py | 3 + web/components.json | 7 +- web/src/components/menu/LiveContextMenu.tsx | 161 +++++++++++++++++- web/src/components/ui/alert.tsx | 59 +++++++ web/src/pages/Settings.tsx | 15 +- web/src/views/live/DraggableGridLayout.tsx | 4 + web/src/views/live/LiveDashboardView.tsx | 1 + .../settings/NotificationsSettingsView.tsx | 60 ++++++- 12 files changed, 351 insertions(+), 22 deletions(-) create mode 100644 web/src/components/ui/alert.tsx diff --git a/.cspell/frigate-dictionary.txt b/.cspell/frigate-dictionary.txt index cc6adcc026..dbab9600ef 100644 --- a/.cspell/frigate-dictionary.txt +++ b/.cspell/frigate-dictionary.txt @@ -44,6 +44,7 @@ codeproject colormap colorspace comms +cooldown coro ctypeslib CUDA diff --git a/docs/docs/configuration/notifications.md b/docs/docs/configuration/notifications.md index 9225ea6e87..8ae2f6d478 100644 --- a/docs/docs/configuration/notifications.md +++ b/docs/docs/configuration/notifications.md @@ -11,14 +11,37 @@ Frigate offers native notifications using the [WebPush Protocol](https://web.dev In order to use notifications the following requirements must be met: -- Frigate must be accessed via a secure https connection +- Frigate must be accessed via a secure `https` connection ([see the authorization docs](/configuration/authentication)). - A supported browser must be used. Currently Chrome, Firefox, and Safari are known to be supported. -- In order for notifications to be usable externally, Frigate must be accessible externally +- In order for notifications to be usable externally, Frigate must be accessible externally. ### Configuration To configure notifications, go to the Frigate WebUI -> Settings -> Notifications and enable, then fill out the fields and save. +Optionally, you can change the default cooldown period for notifications through the `cooldown` parameter in your config file. This parameter can also be overridden at the camera level. + +Notifications will be prevented if either: + +- The global cooldown period hasn't elapsed since any camera's last notification +- The camera-specific cooldown period hasn't elapsed for the specific camera + +```yaml +notifications: + enabled: True + email: "johndoe@gmail.com" + cooldown: 10 # wait 10 seconds before sending another notification from any camera +``` + +```yaml +cameras: + doorbell: + ... + notifications: + enabled: True + cooldown: 30 # wait 30 seconds before sending another notification from the doorbell camera +``` + ### Registration Once notifications are enabled, press the `Register for Notifications` button on all devices that you would like to receive notifications on. This will register the background worker. After this Frigate must be restarted and then notifications will begin to be sent. @@ -39,4 +62,4 @@ Different platforms handle notifications differently, some settings changes may ### Android -Most Android phones have battery optimization settings. To get reliable Notification delivery the browser (Chrome, Firefox) should have battery optimizations disabled. If Frigate is running as a PWA then the Frigate app should have battery optimizations disabled as well. \ No newline at end of file +Most Android phones have battery optimization settings. To get reliable Notification delivery the browser (Chrome, Firefox) should have battery optimizations disabled. If Frigate is running as a PWA then the Frigate app should have battery optimizations disabled as well. diff --git a/docs/docs/configuration/reference.md b/docs/docs/configuration/reference.md index 21b60f4492..b791e708a8 100644 --- a/docs/docs/configuration/reference.md +++ b/docs/docs/configuration/reference.md @@ -420,6 +420,8 @@ notifications: # Optional: Email for push service to reach out to # NOTE: This is required to use notifications email: "admin@example.com" + # Optional: Cooldown time for notifications in seconds (default: shown below) + cooldown: 0 # Optional: Record configuration # NOTE: Can be overridden at the camera level diff --git a/frigate/comms/webpush.py b/frigate/comms/webpush.py index b55b7e82c6..b845c3afdf 100644 --- a/frigate/comms/webpush.py +++ b/frigate/comms/webpush.py @@ -47,6 +47,10 @@ class WebPushClient(Communicator): # type: ignore[misc] self.suspended_cameras: dict[str, int] = { c.name: 0 for c in self.config.cameras.values() } + self.last_camera_notification_time: dict[str, float] = { + c.name: 0 for c in self.config.cameras.values() + } + self.last_notification_time: float = 0 self.notification_queue: queue.Queue[PushNotification] = queue.Queue() self.notification_thread = threading.Thread( target=self._process_notifications, daemon=True @@ -264,6 +268,29 @@ class WebPushClient(Communicator): # type: ignore[misc] ): return + camera: str = payload["after"]["camera"] + current_time = datetime.datetime.now().timestamp() + + # Check global cooldown period + if ( + current_time - self.last_notification_time + < self.config.notifications.cooldown + ): + logger.debug( + f"Skipping notification for {camera} - in global cooldown period" + ) + return + + # Check camera-specific cooldown period + if ( + current_time - self.last_camera_notification_time[camera] + < self.config.cameras[camera].notifications.cooldown + ): + logger.debug( + f"Skipping notification for {camera} - in camera-specific cooldown period" + ) + return + self.check_registrations() state = payload["type"] @@ -278,6 +305,9 @@ class WebPushClient(Communicator): # type: ignore[misc] ): return + self.last_camera_notification_time[camera] = current_time + self.last_notification_time = current_time + reviewId = payload["after"]["id"] sorted_objects: set[str] = set() @@ -287,7 +317,6 @@ class WebPushClient(Communicator): # type: ignore[misc] sorted_objects.update(payload["after"]["data"]["sub_labels"]) - camera: str = payload["after"]["camera"] title = f"{', '.join(sorted_objects).replace('_', ' ').title()}{' was' if state == 'end' else ''} detected in {', '.join(payload['after']['data']['zones']).replace('_', ' ').title()}" message = f"Detected on {camera.replace('_', ' ').title()}" image = f"{payload['after']['thumb_path'].replace('/media/frigate', '')}" diff --git a/frigate/config/camera/notification.py b/frigate/config/camera/notification.py index 79355b8aea..b0d7cebf95 100644 --- a/frigate/config/camera/notification.py +++ b/frigate/config/camera/notification.py @@ -10,6 +10,9 @@ __all__ = ["NotificationConfig"] class NotificationConfig(FrigateBaseModel): enabled: bool = Field(default=False, title="Enable notifications") email: Optional[str] = Field(default=None, title="Email required for push.") + cooldown: Optional[int] = Field( + default=0, ge=0, title="Cooldown period for notifications (time in seconds)." + ) enabled_in_config: Optional[bool] = Field( default=None, title="Keep track of original state of notifications." ) diff --git a/web/components.json b/web/components.json index 053bbcf623..3f112537bb 100644 --- a/web/components.json +++ b/web/components.json @@ -11,6 +11,9 @@ }, "aliases": { "components": "@/components", - "utils": "@/lib/utils" + "utils": "@/lib/utils", + "ui": "@/components/ui", + "lib": "@/lib", + "hooks": "@/hooks" } -} \ No newline at end of file +} diff --git a/web/src/components/menu/LiveContextMenu.tsx b/web/src/components/menu/LiveContextMenu.tsx index 07909a311f..969e647a02 100644 --- a/web/src/components/menu/LiveContextMenu.tsx +++ b/web/src/components/menu/LiveContextMenu.tsx @@ -11,6 +11,9 @@ import { ContextMenuContent, ContextMenuItem, ContextMenuSeparator, + ContextMenuSub, + ContextMenuSubContent, + ContextMenuSubTrigger, ContextMenuTrigger, } from "@/components/ui/context-menu"; import { @@ -24,12 +27,19 @@ import { VolumeSlider } from "@/components/ui/slider"; import { CameraStreamingDialog } from "../settings/CameraStreamingDialog"; import { AllGroupsStreamingSettings, + FrigateConfig, GroupStreamingSettings, } from "@/types/frigateConfig"; import { useStreamingSettings } from "@/context/streaming-settings-provider"; -import { IoIosWarning } from "react-icons/io"; +import { + IoIosNotifications, + IoIosNotificationsOff, + IoIosWarning, +} from "react-icons/io"; import { cn } from "@/lib/utils"; import { useNavigate } from "react-router-dom"; +import { formatUnixTimestampToDateTime } from "@/utils/dateUtil"; +import { useNotifications, useNotificationSuspend } from "@/api/ws"; type LiveContextMenuProps = { className?: string; @@ -48,6 +58,7 @@ type LiveContextMenuProps = { statsState: boolean; toggleStats: () => void; resetPreferredLiveMode: () => void; + config?: FrigateConfig; children?: ReactNode; }; export default function LiveContextMenu({ @@ -67,6 +78,7 @@ export default function LiveContextMenu({ statsState, toggleStats, resetPreferredLiveMode, + config, children, }: LiveContextMenuProps) { const [showSettings, setShowSettings] = useState(false); @@ -185,6 +197,44 @@ export default function LiveContextMenu({ const navigate = useNavigate(); + // notifications + + const notificationsEnabledInConfig = + config?.cameras[camera].notifications.enabled_in_config; + + const { payload: notificationState, send: sendNotification } = + useNotifications(camera); + const { payload: notificationSuspendUntil, send: sendNotificationSuspend } = + useNotificationSuspend(camera); + const [isSuspended, setIsSuspended] = useState(false); + + useEffect(() => { + if (notificationSuspendUntil) { + setIsSuspended( + notificationSuspendUntil !== "0" || notificationState === "OFF", + ); + } + }, [notificationSuspendUntil, notificationState]); + + const handleSuspend = (duration: string) => { + if (duration === "off") { + sendNotification("OFF"); + } else { + sendNotificationSuspend(Number.parseInt(duration)); + } + }; + + const formatSuspendedUntil = (timestamp: string) => { + if (timestamp === "0") return "Frigate restarts."; + + return formatUnixTimestampToDateTime(Number.parseInt(timestamp), { + time_style: "medium", + date_style: "medium", + timezone: config?.ui.timezone, + strftime_fmt: `%b %d, ${config?.ui.time_format == "24hour" ? "%H:%M" : "%I:%M %p"}`, + }); + }; + return (
@@ -288,6 +338,115 @@ export default function LiveContextMenu({ )} + {notificationsEnabledInConfig && ( + <> + + + +
+ Notifications +
+ + +
+
+ {notificationState === "ON" ? ( + <> + {isSuspended ? ( + <> + + Suspended + + ) : ( + <> + + Enabled + + )} + + ) : ( + <> + + Disabled + + )} +
+ {isSuspended && ( + + Until {formatSuspendedUntil(notificationSuspendUntil)} + + )} +
+ + {isSuspended ? ( + <> + + { + sendNotification("ON"); + sendNotificationSuspend(0); + }} + > +
+ {notificationState === "ON" ? ( + Unsuspend + ) : ( + Enable + )} +
+ + + ) : ( + notificationState === "ON" && ( + <> + +
+

+ Suspend for: +

+
+ handleSuspend("5")}> + 5 minutes + + handleSuspend("10")} + > + 10 minutes + + handleSuspend("30")} + > + 30 minutes + + handleSuspend("60")} + > + 1 hour + + handleSuspend("840")} + > + 12 hours + + handleSuspend("1440")} + > + 24 hours + + handleSuspend("off")} + > + Until restart + +
+
+ + ) + )} + + + + )} diff --git a/web/src/components/ui/alert.tsx b/web/src/components/ui/alert.tsx new file mode 100644 index 0000000000..41fa7e0561 --- /dev/null +++ b/web/src/components/ui/alert.tsx @@ -0,0 +1,59 @@ +import * as React from "react" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const alertVariants = cva( + "relative w-full rounded-lg border p-4 [&>svg~*]:pl-7 [&>svg+div]:translate-y-[-3px] [&>svg]:absolute [&>svg]:left-4 [&>svg]:top-4 [&>svg]:text-foreground", + { + variants: { + variant: { + default: "bg-background text-foreground", + destructive: + "border-destructive/50 text-destructive dark:border-destructive [&>svg]:text-destructive", + }, + }, + defaultVariants: { + variant: "default", + }, + } +) + +const Alert = React.forwardRef< + HTMLDivElement, + React.HTMLAttributes & VariantProps +>(({ className, variant, ...props }, ref) => ( +
+)) +Alert.displayName = "Alert" + +const AlertTitle = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +AlertTitle.displayName = "AlertTitle" + +const AlertDescription = React.forwardRef< + HTMLParagraphElement, + React.HTMLAttributes +>(({ className, ...props }, ref) => ( +
+)) +AlertDescription.displayName = "AlertDescription" + +export { Alert, AlertTitle, AlertDescription } diff --git a/web/src/pages/Settings.tsx b/web/src/pages/Settings.tsx index fbf31a00a7..6eeb5bcc33 100644 --- a/web/src/pages/Settings.tsx +++ b/web/src/pages/Settings.tsx @@ -61,19 +61,6 @@ export default function Settings() { const [searchParams] = useSearchParams(); - // available settings views - - const settingsViews = useMemo(() => { - const views = [...allSettingsViews]; - - if (!("Notification" in window) || !window.isSecureContext) { - const index = views.indexOf("notifications"); - views.splice(index, 1); - } - - return views; - }, []); - // TODO: confirm leave page const [unsavedChanges, setUnsavedChanges] = useState(false); const [confirmationDialogOpen, setConfirmationDialogOpen] = useState(false); @@ -160,7 +147,7 @@ export default function Settings() { } }} > - {Object.values(settingsViews).map((item) => ( + {Object.values(allSettingsViews).map((item) => ( resetPreferredLiveMode(camera.name) } + config={config} > void; unmuteAll: () => void; resetPreferredLiveMode: () => void; + config?: FrigateConfig; }; const GridLiveContextMenu = React.forwardRef< @@ -819,6 +821,7 @@ const GridLiveContextMenu = React.forwardRef< muteAll, unmuteAll, resetPreferredLiveMode, + config, ...props }, ref, @@ -849,6 +852,7 @@ const GridLiveContextMenu = React.forwardRef< muteAll={muteAll} unmuteAll={unmuteAll} resetPreferredLiveMode={resetPreferredLiveMode} + config={config} > {children} diff --git a/web/src/views/live/LiveDashboardView.tsx b/web/src/views/live/LiveDashboardView.tsx index 89a2aeef27..45d0d53021 100644 --- a/web/src/views/live/LiveDashboardView.tsx +++ b/web/src/views/live/LiveDashboardView.tsx @@ -507,6 +507,7 @@ export default function LiveDashboardView({ resetPreferredLiveMode={() => resetPreferredLiveMode(camera.name) } + config={config} > (); useEffect(() => { + if (!("Notification" in window) || !window.isSecureContext) { + return; + } navigator.serviceWorker .getRegistration(NOTIFICATION_SERVICE_WORKER) .then((worker) => { @@ -279,6 +283,60 @@ export default function NotificationView({ saveToConfig(values as NotificationSettingsValueType); } + if (!("Notification" in window) || !window.isSecureContext) { + return ( +
+
+
+ + Notification Settings + +
+
+

+ Frigate can natively send push notifications to your device + when it is running in the browser or installed as a PWA. +

+
+ + Read the Documentation{" "} + + +
+
+
+ + + Notifications Unavailable + + + Web push notifications require a secure context ( + https://...). This is a browser limitation. Access + Frigate securely to use notifications. +
+ + Read the Documentation{" "} + + +
+ + +
+
+
+ ); + } + return ( <>
From 124d92daa9f55a99cc33ceefce487968df082cea Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 17 Feb 2025 10:37:17 -0600 Subject: [PATCH 08/19] Improve Object Lifecycle pane (#16635) * add path point tracking to backend * types * draw paths on lifecycle pane * make points clickable * don't display a path if we don't have any saved path points * only object lifecycle points should have a click handler * change to debug log * better debug log message --- frigate/api/event.py | 2 + frigate/events/maintainer.py | 2 + frigate/track/tracked_object.py | 27 +++ .../overlay/detail/ObjectLifecycle.tsx | 207 +++++++++++++++++- web/src/types/event.ts | 1 + web/src/types/timeline.ts | 22 +- 6 files changed, 245 insertions(+), 16 deletions(-) diff --git a/frigate/api/event.py b/frigate/api/event.py index 2473669200..2df32471e8 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -336,6 +336,7 @@ def events_explore(limit: int = 10): "sub_label_score", "average_estimated_speed", "velocity_angle", + "path_data", ] }, "event_count": label_counts[event.label], @@ -622,6 +623,7 @@ def events_search(request: Request, params: EventsSearchQueryParams = Depends()) "sub_label_score", "average_estimated_speed", "velocity_angle", + "path_data", ] } diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index d49da5a97d..fc02dd37ae 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -28,6 +28,7 @@ def should_update_db(prev_event: Event, current_event: Event) -> bool: or prev_event["average_estimated_speed"] != current_event["average_estimated_speed"] or prev_event["velocity_angle"] != current_event["velocity_angle"] + or prev_event["path_data"] != current_event["path_data"] ): return True return False @@ -217,6 +218,7 @@ class EventProcessor(threading.Thread): "velocity_angle": event_data["velocity_angle"], "type": "object", "max_severity": event_data.get("max_severity"), + "path_data": event_data.get("path_data"), }, } diff --git a/frigate/track/tracked_object.py b/frigate/track/tracked_object.py index ac57083df1..0e7464bc2a 100644 --- a/frigate/track/tracked_object.py +++ b/frigate/track/tracked_object.py @@ -2,6 +2,7 @@ import base64 import logging +import math from collections import defaultdict from statistics import median from typing import Optional @@ -66,6 +67,7 @@ class TrackedObject: self.current_estimated_speed = 0 self.average_estimated_speed = 0 self.velocity_angle = 0 + self.path_data = [] self.previous = self.to_dict() @property @@ -148,6 +150,7 @@ class TrackedObject: "attributes": obj_data["attributes"], "current_estimated_speed": self.current_estimated_speed, "velocity_angle": self.velocity_angle, + "path_data": self.path_data, } thumb_update = True @@ -300,6 +303,29 @@ class TrackedObject: if self.obj_data["frame_time"] - self.previous["frame_time"] >= (1 / 3): autotracker_update = True + # update path + width = self.camera_config.detect.width + height = self.camera_config.detect.height + bottom_center = ( + round(obj_data["centroid"][0] / width, 4), + round(obj_data["box"][3] / height, 4), + ) + + # calculate a reasonable movement threshold (e.g., 5% of the frame diagonal) + threshold = 0.05 * math.sqrt(width**2 + height**2) / max(width, height) + + if not self.path_data: + self.path_data.append((bottom_center, obj_data["frame_time"])) + elif ( + math.dist(self.path_data[-1][0], bottom_center) >= threshold + or len(self.path_data) == 1 + ): + # check Euclidean distance before appending + self.path_data.append((bottom_center, obj_data["frame_time"])) + logger.debug( + f"Point tracking: {obj_data['id']}, {bottom_center}, {obj_data['frame_time']}" + ) + self.obj_data.update(obj_data) self.current_zones = current_zones return (thumb_update, significant_change, autotracker_update) @@ -336,6 +362,7 @@ class TrackedObject: "current_estimated_speed": self.current_estimated_speed, "average_estimated_speed": self.average_estimated_speed, "velocity_angle": self.velocity_angle, + "path_data": self.path_data, } if include_thumbnail: diff --git a/web/src/components/overlay/detail/ObjectLifecycle.tsx b/web/src/components/overlay/detail/ObjectLifecycle.tsx index 7481607eb5..656ae275c0 100644 --- a/web/src/components/overlay/detail/ObjectLifecycle.tsx +++ b/web/src/components/overlay/detail/ObjectLifecycle.tsx @@ -11,7 +11,7 @@ import { CarouselPrevious, } from "@/components/ui/carousel"; import { Button } from "@/components/ui/button"; -import { ObjectLifecycleSequence } from "@/types/timeline"; +import { ClassType, ObjectLifecycleSequence } from "@/types/timeline"; import Heading from "@/components/ui/heading"; import { ReviewDetailPaneType } from "@/types/review"; import { FrigateConfig } from "@/types/frigateConfig"; @@ -53,6 +53,13 @@ import { } from "@/components/ui/context-menu"; import { useNavigate } from "react-router-dom"; +type Position = { + x: number; + y: number; + timestamp: number; + lifecycle_item?: ObjectLifecycleSequence; +}; + type ObjectLifecycleProps = { className?: string; event: Event; @@ -108,6 +115,17 @@ export default function ObjectLifecycle({ [config, event], ); + const getObjectColor = useCallback( + (label: string) => { + const objectColor = config?.model?.colormap[label]; + if (objectColor) { + const reversed = [...objectColor].reverse(); + return reversed; + } + }, + [config], + ); + const getZonePolygon = useCallback( (zoneName: string) => { if (!imgRef.current || !config) { @@ -120,7 +138,7 @@ export default function ObjectLifecycle({ return zonePoints .split(",") - .map(parseFloat) + .map(Number.parseFloat) .reduce((acc, value, index) => { const isXCoordinate = index % 2 === 0; const coordinate = isXCoordinate @@ -158,6 +176,43 @@ export default function ObjectLifecycle({ ); }, [config, event.camera]); + const savedPathPoints = useMemo(() => { + return ( + event.data.path_data?.map(([coords, timestamp]: [number[], number]) => ({ + x: coords[0], + y: coords[1], + timestamp, + lifecycle_item: undefined, + })) || [] + ); + }, [event.data.path_data]); + + const eventSequencePoints = useMemo(() => { + return ( + eventSequence + ?.filter((event) => event.data.box !== undefined) + .map((event) => { + const [left, top, width, height] = event.data.box!; + + return { + x: left + width / 2, // Center x-coordinate + y: top + height, // Bottom y-coordinate + timestamp: event.timestamp, + lifecycle_item: event, + }; + }) || [] + ); + }, [eventSequence]); + + // final object path with timeline points included + const pathPoints = useMemo(() => { + // don't display a path if we don't have any saved path points + if (savedPathPoints.length === 0) return []; + return [...savedPathPoints, ...eventSequencePoints].sort( + (a, b) => a.timestamp - b.timestamp, + ); + }, [savedPathPoints, eventSequencePoints]); + const [timeIndex, setTimeIndex] = useState(0); const handleSetBox = useCallback( @@ -171,12 +226,13 @@ export default function ObjectLifecycle({ top: `${box[1] * imgRect.height}px`, width: `${box[2] * imgRect.width}px`, height: `${box[3] * imgRect.height}px`, + borderColor: `rgb(${getObjectColor(event.label)?.join(",")})`, }; setBoxStyle(style); } }, - [imgRef], + [imgRef, event, getObjectColor], ); // image @@ -254,6 +310,21 @@ export default function ObjectLifecycle({ // eslint-disable-next-line react-hooks/exhaustive-deps }, [mainApi, thumbnailApi]); + const handlePathPointClick = useCallback( + (index: number) => { + if (!mainApi || !thumbnailApi || !eventSequence) return; + const sequenceIndex = eventSequence.findIndex( + (item) => item.timestamp === pathPoints[index].timestamp, + ); + if (sequenceIndex !== -1) { + mainApi.scrollTo(sequenceIndex); + thumbnailApi.scrollTo(sequenceIndex); + setCurrent(sequenceIndex); + } + }, + [mainApi, thumbnailApi, eventSequence, pathPoints], + ); + if (!event.id || !eventSequence || !config || !timeIndex) { return ; } @@ -355,13 +426,33 @@ export default function ObjectLifecycle({ ))} {boxStyle && ( -
+
)} + {pathPoints && pathPoints.length > 0 && ( +
+ + + +
+ )} @@ -699,3 +790,105 @@ function getLifecycleItemDescription(lifecycleItem: ObjectLifecycleSequence) { return `${label} detected`; } } + +type ObjectPathProps = { + positions?: Position[]; + color?: number[]; + width?: number; + pointRadius?: number; + imgRef: React.RefObject; + onPointClick?: (index: number) => void; +}; + +const typeColorMap: Partial> = { + [ClassType.VISIBLE]: [0, 255, 0], // Green + [ClassType.GONE]: [255, 0, 0], // Red + [ClassType.ENTERED_ZONE]: [255, 165, 0], // Orange + [ClassType.ATTRIBUTE]: [128, 0, 128], // Purple + [ClassType.ACTIVE]: [255, 255, 0], // Yellow + [ClassType.STATIONARY]: [128, 128, 128], // Gray + [ClassType.HEARD]: [0, 255, 255], // Cyan + [ClassType.EXTERNAL]: [165, 42, 42], // Brown +}; + +function ObjectPath({ + positions, + color = [0, 0, 255], + width = 2, + pointRadius = 4, + imgRef, + onPointClick, +}: ObjectPathProps) { + const getAbsolutePositions = useCallback(() => { + if (!imgRef.current || !positions) return []; + const imgRect = imgRef.current.getBoundingClientRect(); + return positions.map((pos) => ({ + x: pos.x * imgRect.width, + y: pos.y * imgRect.height, + timestamp: pos.timestamp, + lifecycle_item: pos.lifecycle_item, + })); + }, [positions, imgRef]); + + const generateStraightPath = useCallback((points: Position[]) => { + if (!points || points.length < 2) return ""; + let path = `M ${points[0].x} ${points[0].y}`; + for (let i = 1; i < points.length; i++) { + path += ` L ${points[i].x} ${points[i].y}`; + } + return path; + }, []); + + const getPointColor = (baseColor: number[], type?: ClassType) => { + if (type) { + const typeColor = typeColorMap[type]; + if (typeColor) { + return `rgb(${typeColor.join(",")})`; + } + } + // normal path point + return `rgb(${baseColor.map((c) => Math.max(0, c - 10)).join(",")})`; + }; + + if (!imgRef.current) return null; + const absolutePositions = getAbsolutePositions(); + const lineColor = `rgb(${color.join(",")})`; + + return ( + + + {absolutePositions.map((pos, index) => ( + + + + pos.lifecycle_item && onPointClick && onPointClick(index) + } + style={{ cursor: pos.lifecycle_item ? "pointer" : "default" }} + /> + + + + {pos.lifecycle_item + ? getLifecycleItemDescription(pos.lifecycle_item) + : "Tracked point"} + + + + ))} + + ); +} diff --git a/web/src/types/event.ts b/web/src/types/event.ts index 0e7aa99160..d7c8ca6651 100644 --- a/web/src/types/event.ts +++ b/web/src/types/event.ts @@ -22,5 +22,6 @@ export interface Event { area: number; ratio: number; type: "object" | "audio" | "manual"; + path_data: [number[], number][]; }; } diff --git a/web/src/types/timeline.ts b/web/src/types/timeline.ts index 94ef75ebad..66366c2f00 100644 --- a/web/src/types/timeline.ts +++ b/web/src/types/timeline.ts @@ -1,3 +1,15 @@ +export enum ClassType { + VISIBLE = "visible", + GONE = "gone", + ENTERED_ZONE = "entered_zone", + ATTRIBUTE = "attribute", + ACTIVE = "active", + STATIONARY = "stationary", + HEARD = "heard", + EXTERNAL = "external", + PATH_POINT = "path_point", +} + export type ObjectLifecycleSequence = { camera: string; timestamp: number; @@ -10,15 +22,7 @@ export type ObjectLifecycleSequence = { attribute: string; zones: string[]; }; - class_type: - | "visible" - | "gone" - | "entered_zone" - | "attribute" - | "active" - | "stationary" - | "heard" - | "external"; + class_type: ClassType; source_id: string; source: string; }; From b961235187454c0288a750a0debc9b319ec780ea Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 17 Feb 2025 15:49:24 -0600 Subject: [PATCH 09/19] Tracking fixes (#16645) * use config enabled check for ptz cam tracker * ensure we have an object match before accessing score history * add comment for clarity --- frigate/track/norfair_tracker.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/frigate/track/norfair_tracker.py b/frigate/track/norfair_tracker.py index d168bfe946..db17f9313c 100644 --- a/frigate/track/norfair_tracker.py +++ b/frigate/track/norfair_tracker.py @@ -263,12 +263,13 @@ class NorfairTracker(ObjectTracker): # Get the correct tracker for this object's label tracker = self.get_tracker(obj["label"]) - obj["score_history"] = [ - p.data["score"] - for p in next( - (o for o in tracker.tracked_objects if o.global_id == track_id) - ).past_detections - ] + obj_match = next( + (o for o in tracker.tracked_objects if o.global_id == track_id), None + ) + # if we don't have a match, we have a new object + obj["score_history"] = ( + [p.data["score"] for p in obj_match.past_detections] if obj_match else [] + ) self.tracked_objects[id] = obj self.disappeared[id] = 0 self.positions[id] = { @@ -519,7 +520,11 @@ class NorfairTracker(ObjectTracker): default_detections.extend(dets) # Update default tracker with untracked detections - mode = "ptz" if self.ptz_metrics.autotracker_enabled.value else "static" + mode = ( + "ptz" + if self.camera_config.onvif.autotracking.enabled_in_config + else "static" + ) tracked_objects = self.default_tracker[mode].update( detections=default_detections, coord_transformations=coord_transformations ) From 11b1dbf0ff585d84720b7e856149fef5bb2b1e1f Mon Sep 17 00:00:00 2001 From: Ashton Johnson Date: Mon, 17 Feb 2025 16:04:35 -0600 Subject: [PATCH 10/19] Update contributing.md (#16641) Added note to include buildx plugin with Docker --- docs/docs/development/contributing.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/docs/development/contributing.md b/docs/docs/development/contributing.md index 32fc13e1f0..eb33765fe2 100644 --- a/docs/docs/development/contributing.md +++ b/docs/docs/development/contributing.md @@ -34,7 +34,7 @@ Fork [blakeblackshear/frigate-hass-integration](https://github.com/blakeblackshe ### Prerequisites - GNU make -- Docker +- Docker (including buildx plugin) - An extra detector (Coral, OpenVINO, etc.) is optional but recommended to simulate real world performance. :::note From 4f88a5f2ad0968d51390fce0f59a14a8939b5613 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Mon, 17 Feb 2025 17:03:51 -0600 Subject: [PATCH 11/19] Object Lifecycle tweaks (#16648) * Disable object path and add warning for autotracking cameras * clean up --- .../components/overlay/detail/ObjectLifecycle.tsx | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/web/src/components/overlay/detail/ObjectLifecycle.tsx b/web/src/components/overlay/detail/ObjectLifecycle.tsx index 656ae275c0..de343861e9 100644 --- a/web/src/components/overlay/detail/ObjectLifecycle.tsx +++ b/web/src/components/overlay/detail/ObjectLifecycle.tsx @@ -207,11 +207,15 @@ export default function ObjectLifecycle({ // final object path with timeline points included const pathPoints = useMemo(() => { // don't display a path if we don't have any saved path points - if (savedPathPoints.length === 0) return []; + if ( + savedPathPoints.length === 0 || + config?.cameras[event.camera]?.onvif.autotracking.enabled_in_config + ) + return []; return [...savedPathPoints, ...eventSequencePoints].sort( (a, b) => a.timestamp - b.timestamp, ); - }, [savedPathPoints, eventSequencePoints]); + }, [savedPathPoints, eventSequencePoints, config, event]); const [timeIndex, setTimeIndex] = useState(0); @@ -503,6 +507,11 @@ export default function ObjectLifecycle({ {current + 1} of {eventSequence.length}
+ {config?.cameras[event.camera]?.onvif.autotracking.enabled_in_config && ( +
+ Bounding box positions will be inaccurate for autotracking cameras. +
+ )} {showControls && ( Date: Tue, 18 Feb 2025 08:17:51 -0600 Subject: [PATCH 12/19] Reorganize Lifecycle components (#16663) * reorganize lifecycle components * clean up --- .../overlay/detail/ObjectLifecycle.tsx | 206 +++--------------- .../components/overlay/detail/ObjectPath.tsx | 113 ++++++++++ web/src/types/timeline.ts | 11 +- web/src/utils/lifecycleUtil.ts | 47 ++++ 4 files changed, 199 insertions(+), 178 deletions(-) create mode 100644 web/src/components/overlay/detail/ObjectPath.tsx create mode 100644 web/src/utils/lifecycleUtil.ts diff --git a/web/src/components/overlay/detail/ObjectLifecycle.tsx b/web/src/components/overlay/detail/ObjectLifecycle.tsx index de343861e9..40ab543c34 100644 --- a/web/src/components/overlay/detail/ObjectLifecycle.tsx +++ b/web/src/components/overlay/detail/ObjectLifecycle.tsx @@ -11,7 +11,7 @@ import { CarouselPrevious, } from "@/components/ui/carousel"; import { Button } from "@/components/ui/button"; -import { ClassType, ObjectLifecycleSequence } from "@/types/timeline"; +import { ObjectLifecycleSequence } from "@/types/timeline"; import Heading from "@/components/ui/heading"; import { ReviewDetailPaneType } from "@/types/review"; import { FrigateConfig } from "@/types/frigateConfig"; @@ -52,13 +52,8 @@ import { ContextMenuTrigger, } from "@/components/ui/context-menu"; import { useNavigate } from "react-router-dom"; - -type Position = { - x: number; - y: number; - timestamp: number; - lifecycle_item?: ObjectLifecycleSequence; -}; +import { ObjectPath } from "./ObjectPath"; +import { getLifecycleItemDescription } from "@/utils/lifecycleUtil"; type ObjectLifecycleProps = { className?: string; @@ -400,6 +395,8 @@ export default function ObjectLifecycle({ /> {showZones && + imgRef.current?.width && + imgRef.current?.height && lifecycleZones?.map((zone) => (
)} - {pathPoints && pathPoints.length > 0 && ( -
- 0 && ( +
- - -
- )} + + + +
+ )} @@ -755,149 +755,3 @@ export function LifecycleIcon({ return null; } } - -function getLifecycleItemDescription(lifecycleItem: ObjectLifecycleSequence) { - const label = ( - (Array.isArray(lifecycleItem.data.sub_label) - ? lifecycleItem.data.sub_label[0] - : lifecycleItem.data.sub_label) || lifecycleItem.data.label - ).replaceAll("_", " "); - - switch (lifecycleItem.class_type) { - case "visible": - return `${label} detected`; - case "entered_zone": - return `${label} entered ${lifecycleItem.data.zones - .join(" and ") - .replaceAll("_", " ")}`; - case "active": - return `${label} became active`; - case "stationary": - return `${label} became stationary`; - case "attribute": { - let title = ""; - if ( - lifecycleItem.data.attribute == "face" || - lifecycleItem.data.attribute == "license_plate" - ) { - title = `${lifecycleItem.data.attribute.replaceAll( - "_", - " ", - )} detected for ${label}`; - } else { - title = `${ - lifecycleItem.data.label - } recognized as ${lifecycleItem.data.attribute.replaceAll("_", " ")}`; - } - return title; - } - case "gone": - return `${label} left`; - case "heard": - return `${label} heard`; - case "external": - return `${label} detected`; - } -} - -type ObjectPathProps = { - positions?: Position[]; - color?: number[]; - width?: number; - pointRadius?: number; - imgRef: React.RefObject; - onPointClick?: (index: number) => void; -}; - -const typeColorMap: Partial> = { - [ClassType.VISIBLE]: [0, 255, 0], // Green - [ClassType.GONE]: [255, 0, 0], // Red - [ClassType.ENTERED_ZONE]: [255, 165, 0], // Orange - [ClassType.ATTRIBUTE]: [128, 0, 128], // Purple - [ClassType.ACTIVE]: [255, 255, 0], // Yellow - [ClassType.STATIONARY]: [128, 128, 128], // Gray - [ClassType.HEARD]: [0, 255, 255], // Cyan - [ClassType.EXTERNAL]: [165, 42, 42], // Brown -}; - -function ObjectPath({ - positions, - color = [0, 0, 255], - width = 2, - pointRadius = 4, - imgRef, - onPointClick, -}: ObjectPathProps) { - const getAbsolutePositions = useCallback(() => { - if (!imgRef.current || !positions) return []; - const imgRect = imgRef.current.getBoundingClientRect(); - return positions.map((pos) => ({ - x: pos.x * imgRect.width, - y: pos.y * imgRect.height, - timestamp: pos.timestamp, - lifecycle_item: pos.lifecycle_item, - })); - }, [positions, imgRef]); - - const generateStraightPath = useCallback((points: Position[]) => { - if (!points || points.length < 2) return ""; - let path = `M ${points[0].x} ${points[0].y}`; - for (let i = 1; i < points.length; i++) { - path += ` L ${points[i].x} ${points[i].y}`; - } - return path; - }, []); - - const getPointColor = (baseColor: number[], type?: ClassType) => { - if (type) { - const typeColor = typeColorMap[type]; - if (typeColor) { - return `rgb(${typeColor.join(",")})`; - } - } - // normal path point - return `rgb(${baseColor.map((c) => Math.max(0, c - 10)).join(",")})`; - }; - - if (!imgRef.current) return null; - const absolutePositions = getAbsolutePositions(); - const lineColor = `rgb(${color.join(",")})`; - - return ( - - - {absolutePositions.map((pos, index) => ( - - - - pos.lifecycle_item && onPointClick && onPointClick(index) - } - style={{ cursor: pos.lifecycle_item ? "pointer" : "default" }} - /> - - - - {pos.lifecycle_item - ? getLifecycleItemDescription(pos.lifecycle_item) - : "Tracked point"} - - - - ))} - - ); -} diff --git a/web/src/components/overlay/detail/ObjectPath.tsx b/web/src/components/overlay/detail/ObjectPath.tsx new file mode 100644 index 0000000000..d85750ee73 --- /dev/null +++ b/web/src/components/overlay/detail/ObjectPath.tsx @@ -0,0 +1,113 @@ +import { useCallback } from "react"; +import { LifecycleClassType, Position } from "@/types/timeline"; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from "@/components/ui/tooltip"; +import { TooltipPortal } from "@radix-ui/react-tooltip"; +import { getLifecycleItemDescription } from "@/utils/lifecycleUtil"; + +type ObjectPathProps = { + positions?: Position[]; + color?: number[]; + width?: number; + pointRadius?: number; + imgRef: React.RefObject; + onPointClick?: (index: number) => void; +}; + +const typeColorMap: Partial< + Record +> = { + [LifecycleClassType.VISIBLE]: [0, 255, 0], // Green + [LifecycleClassType.GONE]: [255, 0, 0], // Red + [LifecycleClassType.ENTERED_ZONE]: [255, 165, 0], // Orange + [LifecycleClassType.ATTRIBUTE]: [128, 0, 128], // Purple + [LifecycleClassType.ACTIVE]: [255, 255, 0], // Yellow + [LifecycleClassType.STATIONARY]: [128, 128, 128], // Gray + [LifecycleClassType.HEARD]: [0, 255, 255], // Cyan + [LifecycleClassType.EXTERNAL]: [165, 42, 42], // Brown +}; + +export function ObjectPath({ + positions, + color = [0, 0, 255], + width = 2, + pointRadius = 4, + imgRef, + onPointClick, +}: ObjectPathProps) { + const getAbsolutePositions = useCallback(() => { + if (!imgRef.current || !positions) return []; + const imgRect = imgRef.current.getBoundingClientRect(); + return positions.map((pos) => ({ + x: pos.x * imgRect.width, + y: pos.y * imgRect.height, + timestamp: pos.timestamp, + lifecycle_item: pos.lifecycle_item, + })); + }, [positions, imgRef]); + + const generateStraightPath = useCallback((points: Position[]) => { + if (!points || points.length < 2) return ""; + let path = `M ${points[0].x} ${points[0].y}`; + for (let i = 1; i < points.length; i++) { + path += ` L ${points[i].x} ${points[i].y}`; + } + return path; + }, []); + + const getPointColor = (baseColor: number[], type?: LifecycleClassType) => { + if (type) { + const typeColor = typeColorMap[type]; + if (typeColor) { + return `rgb(${typeColor.join(",")})`; + } + } + // normal path point + return `rgb(${baseColor.map((c) => Math.max(0, c - 10)).join(",")})`; + }; + + if (!imgRef.current) return null; + const absolutePositions = getAbsolutePositions(); + const lineColor = `rgb(${color.join(",")})`; + + return ( + + + {absolutePositions.map((pos, index) => ( + + + + pos.lifecycle_item && onPointClick && onPointClick(index) + } + style={{ cursor: pos.lifecycle_item ? "pointer" : "default" }} + /> + + + + {pos.lifecycle_item + ? getLifecycleItemDescription(pos.lifecycle_item) + : "Tracked point"} + + + + ))} + + ); +} diff --git a/web/src/types/timeline.ts b/web/src/types/timeline.ts index 66366c2f00..45a0821ed9 100644 --- a/web/src/types/timeline.ts +++ b/web/src/types/timeline.ts @@ -1,4 +1,4 @@ -export enum ClassType { +export enum LifecycleClassType { VISIBLE = "visible", GONE = "gone", ENTERED_ZONE = "entered_zone", @@ -22,7 +22,7 @@ export type ObjectLifecycleSequence = { attribute: string; zones: string[]; }; - class_type: ClassType; + class_type: LifecycleClassType; source_id: string; source: string; }; @@ -32,3 +32,10 @@ export type TimeRange = { before: number; after: number }; export type TimelineType = "timeline" | "events"; export type TimelineScrubMode = "auto" | "drag" | "hover" | "compat"; + +export type Position = { + x: number; + y: number; + timestamp: number; + lifecycle_item?: ObjectLifecycleSequence; +}; diff --git a/web/src/utils/lifecycleUtil.ts b/web/src/utils/lifecycleUtil.ts new file mode 100644 index 0000000000..f59f3eac9d --- /dev/null +++ b/web/src/utils/lifecycleUtil.ts @@ -0,0 +1,47 @@ +import { ObjectLifecycleSequence } from "@/types/timeline"; + +export function getLifecycleItemDescription( + lifecycleItem: ObjectLifecycleSequence, +) { + const label = ( + (Array.isArray(lifecycleItem.data.sub_label) + ? lifecycleItem.data.sub_label[0] + : lifecycleItem.data.sub_label) || lifecycleItem.data.label + ).replaceAll("_", " "); + + switch (lifecycleItem.class_type) { + case "visible": + return `${label} detected`; + case "entered_zone": + return `${label} entered ${lifecycleItem.data.zones + .join(" and ") + .replaceAll("_", " ")}`; + case "active": + return `${label} became active`; + case "stationary": + return `${label} became stationary`; + case "attribute": { + let title = ""; + if ( + lifecycleItem.data.attribute == "face" || + lifecycleItem.data.attribute == "license_plate" + ) { + title = `${lifecycleItem.data.attribute.replaceAll( + "_", + " ", + )} detected for ${label}`; + } else { + title = `${ + lifecycleItem.data.label + } recognized as ${lifecycleItem.data.attribute.replaceAll("_", " ")}`; + } + return title; + } + case "gone": + return `${label} left`; + case "heard": + return `${label} heard`; + case "external": + return `${label} detected`; + } +} From 5bd412071ad37c06d5e63275b4cf4f28d43cd214 Mon Sep 17 00:00:00 2001 From: Lander Noterman Date: Tue, 18 Feb 2025 15:38:07 +0100 Subject: [PATCH 13/19] Add support for JetPack 6 (#16571) * it builds! * some fixes * use python 3.11 (rc1) * add deadsnakes ppa for more recent python 3.11 in jetson images * fix pip stuff * revert to tensor 8.6 * revert changes to docker/main * add hook to install deadsnakes ppa for tensorrt/jetson * remove unnecessary pip break-system-packages * move tflite_runtime to requirements-wheels.txt --- .github/workflows/ci.yml | 29 ++++++++++++++++ docker/main/Dockerfile | 33 ++++++++++++++----- docker/main/install_deps.sh | 13 ++------ docker/main/requirements-wheels.txt | 5 ++- docker/rockchip/Dockerfile | 3 +- docker/tensorrt/Dockerfile.amd64 | 4 +-- docker/tensorrt/Dockerfile.arm64 | 29 ++++++++++++---- docker/tensorrt/build_jetson_ffmpeg.sh | 17 ++++++++-- .../detector/build_python_tensorrt.sh | 14 ++++---- docker/tensorrt/requirements-arm64.txt | 2 +- docker/tensorrt/trt.hcl | 13 +++++++- docker/tensorrt/trt.mk | 12 +++++++ 12 files changed, 132 insertions(+), 42 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2046ed1003..c9b34c62fb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,6 +136,35 @@ jobs: tensorrt.tags=${{ steps.setup.outputs.image-name }}-tensorrt-jp5 *.cache-from=type=registry,ref=${{ steps.setup.outputs.cache-name }}-jp5 *.cache-to=type=registry,ref=${{ steps.setup.outputs.cache-name }}-jp5,mode=max + jetson_jp6_build: + runs-on: ubuntu-22.04 + name: Jetson Jetpack 6 + steps: + - name: Check out code + uses: actions/checkout@v4 + with: + persist-credentials: false + - name: Set up QEMU and Buildx + id: setup + uses: ./.github/actions/setup + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push TensorRT (Jetson, Jetpack 6) + env: + ARCH: arm64 + BASE_IMAGE: nvcr.io/nvidia/tensorrt:23.12-py3-igpu + SLIM_BASE: nvcr.io/nvidia/tensorrt:23.12-py3-igpu + TRT_BASE: nvcr.io/nvidia/tensorrt:23.12-py3-igpu + uses: docker/bake-action@v6 + with: + source: . + push: true + targets: tensorrt + files: docker/tensorrt/trt.hcl + set: | + tensorrt.tags=${{ steps.setup.outputs.image-name }}-tensorrt-jp6 + *.cache-from=type=registry,ref=${{ steps.setup.outputs.cache-name }}-jp6 + *.cache-to=type=registry,ref=${{ steps.setup.outputs.cache-name }}-jp6,mode=max amd64_extra_builds: runs-on: ubuntu-22.04 name: AMD64 Extra Build diff --git a/docker/main/Dockerfile b/docker/main/Dockerfile index 4c3416789f..0bafeab80d 100644 --- a/docker/main/Dockerfile +++ b/docker/main/Dockerfile @@ -3,14 +3,27 @@ # https://askubuntu.com/questions/972516/debian-frontend-environment-variable ARG DEBIAN_FRONTEND=noninteractive +# Globally set pip break-system-packages option to avoid having to specify it every time +ARG PIP_BREAK_SYSTEM_PACKAGES=1 + ARG BASE_IMAGE=debian:12 ARG SLIM_BASE=debian:12-slim +# A hook that allows us to inject commands right after the base images +ARG BASE_HOOK= + FROM ${BASE_IMAGE} AS base +ARG PIP_BREAK_SYSTEM_PACKAGES + +RUN ${BASE_HOOK} FROM --platform=${BUILDPLATFORM} debian:12 AS base_host +ARG PIP_BREAK_SYSTEM_PACKAGES FROM ${SLIM_BASE} AS slim-base +ARG PIP_BREAK_SYSTEM_PACKAGES + +RUN ${BASE_HOOK} FROM slim-base AS wget ARG DEBIAN_FRONTEND @@ -66,8 +79,8 @@ COPY docker/main/requirements-ov.txt /requirements-ov.txt RUN apt-get -qq update \ && apt-get -qq install -y wget python3 python3-dev python3-distutils gcc pkg-config libhdf5-dev \ && wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ - && python3 get-pip.py "pip" --break-system-packages \ - && pip install --break-system-packages -r /requirements-ov.txt + && python3 get-pip.py "pip" \ + && pip install -r /requirements-ov.txt # Get OpenVino Model RUN --mount=type=bind,source=docker/main/build_ov_model.py,target=/build_ov_model.py \ @@ -142,8 +155,8 @@ RUN apt-get -qq update \ apt-transport-https wget \ && apt-get -qq update \ && apt-get -qq install -y \ - python3 \ - python3-dev \ + python3.11 \ + python3.11-dev \ # opencv dependencies build-essential cmake git pkg-config libgtk-3-dev \ libavcodec-dev libavformat-dev libswscale-dev libv4l-dev \ @@ -157,11 +170,13 @@ RUN apt-get -qq update \ gcc gfortran libopenblas-dev liblapack-dev && \ rm -rf /var/lib/apt/lists/* +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 + RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ - && python3 get-pip.py "pip" --break-system-packages + && python3 get-pip.py "pip" COPY docker/main/requirements.txt /requirements.txt -RUN pip3 install -r /requirements.txt --break-system-packages +RUN pip3 install -r /requirements.txt # Build pysqlite3 from source COPY docker/main/build_pysqlite3.sh /build_pysqlite3.sh @@ -215,8 +230,8 @@ RUN --mount=type=bind,source=docker/main/install_deps.sh,target=/deps/install_de /deps/install_deps.sh RUN --mount=type=bind,from=wheels,source=/wheels,target=/deps/wheels \ - python3 -m pip install --upgrade pip --break-system-packages && \ - pip3 install -U /deps/wheels/*.whl --break-system-packages + python3 -m pip install --upgrade pip && \ + pip3 install -U /deps/wheels/*.whl COPY --from=deps-rootfs / / @@ -263,7 +278,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* RUN --mount=type=bind,source=./docker/main/requirements-dev.txt,target=/workspace/frigate/requirements-dev.txt \ - pip3 install -r requirements-dev.txt --break-system-packages + pip3 install -r requirements-dev.txt HEALTHCHECK NONE diff --git a/docker/main/install_deps.sh b/docker/main/install_deps.sh index ee84f6a141..a7b7789c0d 100755 --- a/docker/main/install_deps.sh +++ b/docker/main/install_deps.sh @@ -11,7 +11,7 @@ apt-get -qq install --no-install-recommends -y \ lbzip2 \ procps vainfo \ unzip locales tzdata libxml2 xz-utils \ - python3 \ + python3.11 \ python3-pip \ curl \ lsof \ @@ -21,6 +21,8 @@ apt-get -qq install --no-install-recommends -y \ libglib2.0-0 \ libusb-1.0.0 +update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 + mkdir -p -m 600 /root/.gnupg # install coral runtime @@ -29,15 +31,6 @@ unset DEBIAN_FRONTEND yes | dpkg -i /tmp/libedgetpu1-max.deb && export DEBIAN_FRONTEND=noninteractive rm /tmp/libedgetpu1-max.deb -# install python3 & tflite runtime -if [[ "${TARGETARCH}" == "amd64" ]]; then - pip3 install --break-system-packages https://github.com/frigate-nvr/TFlite-builds/releases/download/v2.17.1/tflite_runtime-2.17.1-cp311-cp311-linux_x86_64.whl -fi - -if [[ "${TARGETARCH}" == "arm64" ]]; then - pip3 install --break-system-packages https://github.com/feranick/TFlite-builds/releases/download/v2.17.1/tflite_runtime-2.17.1-cp311-cp311-linux_aarch64.whl -fi - # btbn-ffmpeg -> amd64 if [[ "${TARGETARCH}" == "amd64" ]]; then mkdir -p /usr/lib/ffmpeg/5.0 diff --git a/docker/main/requirements-wheels.txt b/docker/main/requirements-wheels.txt index e43e741558..f06f82d886 100644 --- a/docker/main/requirements-wheels.txt +++ b/docker/main/requirements-wheels.txt @@ -68,4 +68,7 @@ netaddr==0.8.* netifaces==0.10.* verboselogs==1.7.* virtualenv==20.17.* -prometheus-client == 0.21.* \ No newline at end of file +prometheus-client == 0.21.* +# TFLite +tflite_runtime @ https://github.com/feranick/TFlite-builds/releases/download/v2.17.1/tflite_runtime-2.17.1-cp311-cp311-linux_x86_64.whl; platform_machine == 'x86_64' +tflite_runtime @ https://github.com/feranick/TFlite-builds/releases/download/v2.17.1/tflite_runtime-2.17.1-cp311-cp311-linux_aarch64.whl; platform_machine == 'aarch64' diff --git a/docker/rockchip/Dockerfile b/docker/rockchip/Dockerfile index e9c9602a8e..09380dfb3b 100644 --- a/docker/rockchip/Dockerfile +++ b/docker/rockchip/Dockerfile @@ -8,7 +8,6 @@ COPY docker/main/requirements-wheels.txt /requirements-wheels.txt COPY docker/rockchip/requirements-wheels-rk.txt /requirements-wheels-rk.txt RUN sed -i "/https:\/\//d" /requirements-wheels.txt RUN sed -i "/onnxruntime/d" /requirements-wheels.txt -RUN python3 -m pip config set global.break-system-packages true RUN pip3 wheel --wheel-dir=/rk-wheels -c /requirements-wheels.txt -r /requirements-wheels-rk.txt RUN rm -rf /rk-wheels/opencv_python-* @@ -16,7 +15,7 @@ FROM deps AS rk-frigate ARG TARGETARCH RUN --mount=type=bind,from=rk-wheels,source=/rk-wheels,target=/deps/rk-wheels \ - pip3 install --no-deps -U /deps/rk-wheels/*.whl --break-system-packages + pip3 install --no-deps -U /deps/rk-wheels/*.whl WORKDIR /opt/frigate/ COPY --from=rootfs / / diff --git a/docker/tensorrt/Dockerfile.amd64 b/docker/tensorrt/Dockerfile.amd64 index 276094ed21..6be11c2104 100644 --- a/docker/tensorrt/Dockerfile.amd64 +++ b/docker/tensorrt/Dockerfile.amd64 @@ -17,7 +17,7 @@ FROM tensorrt-base AS frigate-tensorrt ENV TRT_VER=8.6.1 RUN python3 -m pip config set global.break-system-packages true RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \ - pip3 install -U /deps/trt-wheels/*.whl --break-system-packages && \ + pip3 install -U /deps/trt-wheels/*.whl && \ ldconfig WORKDIR /opt/frigate/ @@ -32,4 +32,4 @@ COPY --from=trt-deps /usr/local/cuda-12.1 /usr/local/cuda COPY docker/tensorrt/detector/rootfs/ / COPY --from=trt-deps /usr/local/lib/libyolo_layer.so /usr/local/lib/libyolo_layer.so RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \ - pip3 install -U /deps/trt-wheels/*.whl --break-system-packages + pip3 install -U /deps/trt-wheels/*.whl diff --git a/docker/tensorrt/Dockerfile.arm64 b/docker/tensorrt/Dockerfile.arm64 index ba2638fcb6..33fd8182a3 100644 --- a/docker/tensorrt/Dockerfile.arm64 +++ b/docker/tensorrt/Dockerfile.arm64 @@ -7,20 +7,25 @@ ARG BASE_IMAGE FROM ${BASE_IMAGE} AS build-wheels ARG DEBIAN_FRONTEND +# Add deadsnakes PPA for python3.11 +RUN apt-get -qq update && \ + apt-get -qq install -y --no-install-recommends \ + software-properties-common \ + && add-apt-repository ppa:deadsnakes/ppa + # Use a separate container to build wheels to prevent build dependencies in final image RUN apt-get -qq update \ && apt-get -qq install -y --no-install-recommends \ - python3.9 python3.9-dev \ + python3.11 python3.11-dev \ wget build-essential cmake git \ && rm -rf /var/lib/apt/lists/* -# Ensure python3 defaults to python3.9 -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 1 +# Ensure python3 defaults to python3.11 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 RUN wget -q https://bootstrap.pypa.io/get-pip.py -O get-pip.py \ && python3 get-pip.py "pip" - FROM build-wheels AS trt-wheels ARG DEBIAN_FRONTEND ARG TARGETARCH @@ -41,11 +46,12 @@ RUN --mount=type=bind,source=docker/tensorrt/detector/build_python_tensorrt.sh,t && TENSORRT_VER=$(cat /etc/TENSORRT_VER) /deps/build_python_tensorrt.sh COPY docker/tensorrt/requirements-arm64.txt /requirements-tensorrt.txt -ADD https://nvidia.box.com/shared/static/psl23iw3bh7hlgku0mjo1xekxpego3e3.whl /tmp/onnxruntime_gpu-1.15.1-cp311-cp311-linux_aarch64.whl +# See https://elinux.org/Jetson_Zoo#ONNX_Runtime +ADD https://nvidia.box.com/shared/static/9yvw05k6u343qfnkhdv2x6xhygze0aq1.whl /tmp/onnxruntime_gpu-1.19.0-cp311-cp311-linux_aarch64.whl RUN pip3 uninstall -y onnxruntime-openvino \ && pip3 wheel --wheel-dir=/trt-wheels -r /requirements-tensorrt.txt \ - && pip3 install --no-deps /tmp/onnxruntime_gpu-1.15.1-cp311-cp311-linux_aarch64.whl + && pip3 install --no-deps /tmp/onnxruntime_gpu-1.19.0-cp311-cp311-linux_aarch64.whl FROM build-wheels AS trt-model-wheels ARG DEBIAN_FRONTEND @@ -67,12 +73,18 @@ RUN --mount=type=bind,source=docker/tensorrt/build_jetson_ffmpeg.sh,target=/deps # Frigate w/ TensorRT for NVIDIA Jetson platforms FROM tensorrt-base AS frigate-tensorrt RUN apt-get update \ - && apt-get install -y python-is-python3 libprotobuf17 \ + && apt-get install -y python-is-python3 libprotobuf23 \ && rm -rf /var/lib/apt/lists/* RUN rm -rf /usr/lib/btbn-ffmpeg/ COPY --from=jetson-ffmpeg /rootfs / +# ffmpeg runtime dependencies +RUN apt-get -qq update \ + && apt-get -qq install -y --no-install-recommends \ + libx264-163 libx265-199 libegl1 \ + && rm -rf /var/lib/apt/lists/* + COPY --from=trt-wheels /etc/TENSORRT_VER /etc/TENSORRT_VER RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \ --mount=type=bind,from=trt-model-wheels,source=/trt-model-wheels,target=/deps/trt-model-wheels \ @@ -81,3 +93,6 @@ RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels WORKDIR /opt/frigate/ COPY --from=rootfs / / + +# Fixes "Error importing detector runtime: /usr/lib/aarch64-linux-gnu/libstdc++.so.6: cannot allocate memory in static TLS block" +ENV LD_PRELOAD /usr/lib/aarch64-linux-gnu/libstdc++.so.6 diff --git a/docker/tensorrt/build_jetson_ffmpeg.sh b/docker/tensorrt/build_jetson_ffmpeg.sh index f4e55c2bb3..692612137a 100755 --- a/docker/tensorrt/build_jetson_ffmpeg.sh +++ b/docker/tensorrt/build_jetson_ffmpeg.sh @@ -14,14 +14,27 @@ apt-get -qq install -y --no-install-recommends libx264-dev libx265-dev pushd /tmp # Install libnvmpi to enable nvmpi decoders (h264_nvmpi, hevc_nvmpi) -if [ -e /usr/local/cuda-10.2 ]; then +if [ -e /usr/local/cuda-12 ]; then + # assume Jetpack 6.2 + apt-key adv --fetch-key https://repo.download.nvidia.com/jetson/jetson-ota-public.asc + echo "deb https://repo.download.nvidia.com/jetson/common r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list + echo "deb https://repo.download.nvidia.com/jetson/t234 r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list + echo "deb https://repo.download.nvidia.com/jetson/ffmpeg r36.4 main" >> /etc/apt/sources.list.d/nvidia-l4t-apt-source.list + + mkdir -p /opt/nvidia/l4t-packages/ + touch /opt/nvidia/l4t-packages/.nv-l4t-disable-boot-fw-update-in-preinstall + + apt-get update + apt-get -qq install -y --no-install-recommends -o Dpkg::Options::="--force-confold" nvidia-l4t-jetson-multimedia-api +elif [ -e /usr/local/cuda-10.2 ]; then # assume Jetpack 4.X wget -q https://developer.nvidia.com/embedded/L4T/r32_Release_v5.0/T186/Jetson_Multimedia_API_R32.5.0_aarch64.tbz2 -O jetson_multimedia_api.tbz2 + tar xaf jetson_multimedia_api.tbz2 -C / && rm jetson_multimedia_api.tbz2 else # assume Jetpack 5.X wget -q https://developer.nvidia.com/downloads/embedded/l4t/r35_release_v3.1/release/jetson_multimedia_api_r35.3.1_aarch64.tbz2 -O jetson_multimedia_api.tbz2 + tar xaf jetson_multimedia_api.tbz2 -C / && rm jetson_multimedia_api.tbz2 fi -tar xaf jetson_multimedia_api.tbz2 -C / && rm jetson_multimedia_api.tbz2 wget -q https://github.com/AndBobsYourUncle/jetson-ffmpeg/archive/9c17b09.zip -O jetson-ffmpeg.zip unzip jetson-ffmpeg.zip && rm jetson-ffmpeg.zip && mv jetson-ffmpeg-* jetson-ffmpeg && cd jetson-ffmpeg diff --git a/docker/tensorrt/detector/build_python_tensorrt.sh b/docker/tensorrt/detector/build_python_tensorrt.sh index 21b6ae268f..325103485b 100755 --- a/docker/tensorrt/detector/build_python_tensorrt.sh +++ b/docker/tensorrt/detector/build_python_tensorrt.sh @@ -6,23 +6,23 @@ mkdir -p /trt-wheels if [[ "${TARGETARCH}" == "arm64" ]]; then - # NVIDIA supplies python-tensorrt for python3.8, but frigate uses python3.9, + # NVIDIA supplies python-tensorrt for python3.10, but frigate uses python3.11, # so we must build python-tensorrt ourselves. # Get python-tensorrt source - mkdir /workspace + mkdir -p /workspace cd /workspace - git clone -b ${TENSORRT_VER} https://github.com/NVIDIA/TensorRT.git --depth=1 + git clone -b release/8.6 https://github.com/NVIDIA/TensorRT.git --depth=1 # Collect dependencies EXT_PATH=/workspace/external && mkdir -p $EXT_PATH - pip3 install pybind11 && ln -s /usr/local/lib/python3.9/dist-packages/pybind11 $EXT_PATH/pybind11 - ln -s /usr/include/python3.9 $EXT_PATH/python3.9 + pip3 install pybind11 && ln -s /usr/local/lib/python3.11/dist-packages/pybind11 $EXT_PATH/pybind11 + ln -s /usr/include/python3.11 $EXT_PATH/python3.11 ln -s /usr/include/aarch64-linux-gnu/NvOnnxParser.h /workspace/TensorRT/parsers/onnx/ # Build wheel cd /workspace/TensorRT/python - EXT_PATH=$EXT_PATH PYTHON_MAJOR_VERSION=3 PYTHON_MINOR_VERSION=9 TARGET_ARCHITECTURE=aarch64 /bin/bash ./build.sh - mv build/dist/*.whl /trt-wheels/ + EXT_PATH=$EXT_PATH PYTHON_MAJOR_VERSION=3 PYTHON_MINOR_VERSION=11 TARGET_ARCHITECTURE=aarch64 TENSORRT_MODULE=tensorrt /bin/bash ./build.sh + mv build/bindings_wheel/dist/*.whl /trt-wheels/ fi diff --git a/docker/tensorrt/requirements-arm64.txt b/docker/tensorrt/requirements-arm64.txt index 67489f80b9..c9b6181803 100644 --- a/docker/tensorrt/requirements-arm64.txt +++ b/docker/tensorrt/requirements-arm64.txt @@ -1 +1 @@ -cuda-python == 11.7; platform_machine == 'aarch64' \ No newline at end of file +cuda-python == 12.6.*; platform_machine == 'aarch64' diff --git a/docker/tensorrt/trt.hcl b/docker/tensorrt/trt.hcl index 3195fb5bf5..730f54053c 100644 --- a/docker/tensorrt/trt.hcl +++ b/docker/tensorrt/trt.hcl @@ -13,13 +13,24 @@ variable "TRT_BASE" { variable "COMPUTE_LEVEL" { default = "" } +variable "BASE_HOOK" { + # Ensure an up-to-date python 3.11 is available in tensorrt/jetson image + default = < Date: Tue, 18 Feb 2025 07:46:29 -0700 Subject: [PATCH 14/19] Remove thumb from database field (#16647) * Remove thumbnail from dict * Create thumbnail diectory * Cleanup handling of tracked object images * Make thumbnail optional * Handle cases where thumbnail is used * Expand options for thumbnail api * Fix up the does not exist condition * Remove absolute usages of thumbnails * Write thumbnails for external events * Reduce webp quality * Use webp everywhere in frontend * Formatting * Always consider all events when re-indexing * Add thumbnail deletion and cleanup path management * Cleanup imports * Rename def * Don't save thumbnail for every object * Correct event count * Use correct function * Include thumbnail in query * Remove unused * Fix requiring exception --- frigate/api/media.py | 35 ++++-- frigate/app.py | 2 + frigate/const.py | 1 + frigate/embeddings/embeddings.py | 26 ++--- frigate/embeddings/maintainer.py | 5 +- frigate/events/cleanup.py | 38 +++---- frigate/events/external.py | 17 ++- frigate/events/maintainer.py | 5 +- frigate/object_processing.py | 52 +++------ frigate/track/tracked_object.py | 100 ++++++++++++++---- frigate/util/path.py | 51 +++++++++ migrations/028_optional_event_thumbnail.py | 36 +++++++ web/src/components/card/SearchThumbnail.tsx | 2 +- .../overlay/detail/ReviewDetailDialog.tsx | 4 +- .../overlay/detail/SearchDetailDialog.tsx | 2 +- web/src/views/explore/ExploreView.tsx | 2 +- 16 files changed, 241 insertions(+), 137 deletions(-) create mode 100644 frigate/util/path.py create mode 100644 migrations/028_optional_event_thumbnail.py diff --git a/frigate/api/media.py b/frigate/api/media.py index a9455919ba..74e9e7aaa3 100644 --- a/frigate/api/media.py +++ b/frigate/api/media.py @@ -1,6 +1,5 @@ """Image and video apis.""" -import base64 import glob import logging import os @@ -40,6 +39,7 @@ from frigate.models import Event, Previews, Recordings, Regions, ReviewSegment from frigate.object_processing import TrackedObjectProcessor from frigate.util.builtin import get_tz_modifiers from frigate.util.image import get_image_from_recording +from frigate.util.path import get_event_thumbnail_bytes logger = logging.getLogger(__name__) @@ -804,10 +804,11 @@ def event_snapshot( ) -@router.get("/events/{event_id}/thumbnail.jpg") +@router.get("/events/{event_id}/thumbnail.{extension}") def event_thumbnail( request: Request, event_id: str, + extension: str, max_cache_age: int = Query( 2592000, description="Max cache age in seconds. Default 30 days in seconds." ), @@ -816,11 +817,15 @@ def event_thumbnail( thumbnail_bytes = None event_complete = False try: - event = Event.get(Event.id == event_id) + event: Event = Event.get(Event.id == event_id) if event.end_time is not None: event_complete = True - thumbnail_bytes = base64.b64decode(event.thumbnail) + + thumbnail_bytes = get_event_thumbnail_bytes(event) except DoesNotExist: + thumbnail_bytes = None + + if thumbnail_bytes is None: # see if the object is currently being tracked try: camera_states = request.app.detected_frames_processor.camera_states.values() @@ -828,7 +833,7 @@ def event_thumbnail( if event_id in camera_state.tracked_objects: tracked_obj = camera_state.tracked_objects.get(event_id) if tracked_obj is not None: - thumbnail_bytes = tracked_obj.get_thumbnail() + thumbnail_bytes = tracked_obj.get_thumbnail(extension) except Exception: return JSONResponse( content={"success": False, "message": "Event not found"}, @@ -843,8 +848,8 @@ def event_thumbnail( # android notifications prefer a 2:1 ratio if format == "android": - jpg_as_np = np.frombuffer(thumbnail_bytes, dtype=np.uint8) - img = cv2.imdecode(jpg_as_np, flags=1) + img_as_np = np.frombuffer(thumbnail_bytes, dtype=np.uint8) + img = cv2.imdecode(img_as_np, flags=1) thumbnail = cv2.copyMakeBorder( img, 0, @@ -854,17 +859,25 @@ def event_thumbnail( cv2.BORDER_CONSTANT, (0, 0, 0), ) - ret, jpg = cv2.imencode(".jpg", thumbnail, [int(cv2.IMWRITE_JPEG_QUALITY), 70]) - thumbnail_bytes = jpg.tobytes() + + quality_params = None + + if extension == "jpg" or extension == "jpeg": + quality_params = [int(cv2.IMWRITE_JPEG_QUALITY), 70] + elif extension == "webp": + quality_params = [int(cv2.IMWRITE_WEBP_QUALITY), 60] + + _, img = cv2.imencode(f".{img}", thumbnail, quality_params) + thumbnail_bytes = img.tobytes() return Response( thumbnail_bytes, - media_type="image/jpeg", + media_type=f"image/{extension}", headers={ "Cache-Control": f"private, max-age={max_cache_age}" if event_complete else "no-store", - "Content-Type": "image/jpeg", + "Content-Type": f"image/{extension}", }, ) diff --git a/frigate/app.py b/frigate/app.py index 6ff4a1a412..400d4bca0f 100644 --- a/frigate/app.py +++ b/frigate/app.py @@ -39,6 +39,7 @@ from frigate.const import ( MODEL_CACHE_DIR, RECORD_DIR, SHM_FRAMES_VAR, + THUMB_DIR, ) from frigate.data_processing.types import DataProcessorMetrics from frigate.db.sqlitevecq import SqliteVecQueueDatabase @@ -105,6 +106,7 @@ class FrigateApp: dirs = [ CONFIG_DIR, RECORD_DIR, + THUMB_DIR, f"{CLIPS_DIR}/cache", CACHE_DIR, MODEL_CACHE_DIR, diff --git a/frigate/const.py b/frigate/const.py index 16df8b887e..eb48e9bf9e 100644 --- a/frigate/const.py +++ b/frigate/const.py @@ -7,6 +7,7 @@ BASE_DIR = "/media/frigate" CLIPS_DIR = f"{BASE_DIR}/clips" EXPORT_DIR = f"{BASE_DIR}/exports" FACE_DIR = f"{CLIPS_DIR}/faces" +THUMB_DIR = f"{CLIPS_DIR}/thumbs" RECORD_DIR = f"{BASE_DIR}/recordings" BIRDSEYE_PIPE = "/tmp/cache/birdseye" CACHE_DIR = "/tmp/cache" diff --git a/frigate/embeddings/embeddings.py b/frigate/embeddings/embeddings.py index d8a4a2f4d0..5ce7ba86dd 100644 --- a/frigate/embeddings/embeddings.py +++ b/frigate/embeddings/embeddings.py @@ -1,6 +1,5 @@ """SQLite-vec embeddings database.""" -import base64 import datetime import logging import os @@ -21,6 +20,7 @@ from frigate.db.sqlitevecq import SqliteVecQueueDatabase from frigate.models import Event from frigate.types import ModelStatusTypesEnum from frigate.util.builtin import serialize +from frigate.util.path import get_event_thumbnail_bytes from .functions.onnx import GenericONNXEmbedding, ModelTypeEnum @@ -264,14 +264,7 @@ class Embeddings: st = time.time() # Get total count of events to process - total_events = ( - Event.select() - .where( - (Event.has_clip == True | Event.has_snapshot == True) - & Event.thumbnail.is_null(False) - ) - .count() - ) + total_events = Event.select().count() batch_size = 32 current_page = 1 @@ -289,10 +282,6 @@ class Embeddings: events = ( Event.select() - .where( - (Event.has_clip == True | Event.has_snapshot == True) - & Event.thumbnail.is_null(False) - ) .order_by(Event.start_time.desc()) .paginate(current_page, batch_size) ) @@ -302,7 +291,12 @@ class Embeddings: batch_thumbs = {} batch_descs = {} for event in events: - batch_thumbs[event.id] = base64.b64decode(event.thumbnail) + thumbnail = get_event_thumbnail_bytes(event) + + if thumbnail is None: + continue + + batch_thumbs[event.id] = thumbnail totals["thumbnails"] += 1 if description := event.data.get("description", "").strip(): @@ -341,10 +335,6 @@ class Embeddings: current_page += 1 events = ( Event.select() - .where( - (Event.has_clip == True | Event.has_snapshot == True) - & Event.thumbnail.is_null(False) - ) .order_by(Event.start_time.desc()) .paginate(current_page, batch_size) ) diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index b7623722d5..7925345b22 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -38,6 +38,7 @@ from frigate.models import Event from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import serialize from frigate.util.image import SharedMemoryFrameManager, calculate_region +from frigate.util.path import get_event_thumbnail_bytes from .embeddings import Embeddings @@ -215,7 +216,7 @@ class EmbeddingMaintainer(threading.Thread): continue # Extract valid thumbnail - thumbnail = base64.b64decode(event.thumbnail) + thumbnail = get_event_thumbnail_bytes(event) # Embed the thumbnail self._embed_thumbnail(event_id, thumbnail) @@ -390,7 +391,7 @@ class EmbeddingMaintainer(threading.Thread): logger.error(f"GenAI not enabled for camera {event.camera}") return - thumbnail = base64.b64decode(event.thumbnail) + thumbnail = get_event_thumbnail_bytes(event) logger.debug( f"Trying {source} regeneration for {event}, has_snapshot: {event.has_snapshot}" diff --git a/frigate/events/cleanup.py b/frigate/events/cleanup.py index d4efb26e83..ae39e3fd20 100644 --- a/frigate/events/cleanup.py +++ b/frigate/events/cleanup.py @@ -11,6 +11,7 @@ from frigate.config import FrigateConfig from frigate.const import CLIPS_DIR from frigate.db.sqlitevecq import SqliteVecQueueDatabase from frigate.models import Event, Timeline +from frigate.util.path import delete_event_images logger = logging.getLogger(__name__) @@ -64,7 +65,6 @@ class EventCleanup(threading.Thread): def expire_snapshots(self) -> list[str]: ## Expire events from unlisted cameras based on the global config retain_config = self.config.snapshots.retain - file_extension = "jpg" update_params = {"has_snapshot": False} distinct_labels = self.get_removed_camera_labels() @@ -83,6 +83,7 @@ class EventCleanup(threading.Thread): Event.select( Event.id, Event.camera, + Event.thumbnail, ) .where( Event.camera.not_in(self.camera_keys), @@ -94,22 +95,15 @@ class EventCleanup(threading.Thread): .iterator() ) logger.debug(f"{len(list(expired_events))} events can be expired") + # delete the media from disk for expired in expired_events: - media_name = f"{expired.camera}-{expired.id}" - media_path = Path( - f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}" - ) + deleted = delete_event_images(expired) - try: - media_path.unlink(missing_ok=True) - if file_extension == "jpg": - media_path = Path( - f"{os.path.join(CLIPS_DIR, media_name)}-clean.png" - ) - media_path.unlink(missing_ok=True) - except OSError as e: - logger.warning(f"Unable to delete event images: {e}") + if not deleted: + logger.warning( + f"Unable to delete event images for {expired.camera}: {expired.id}" + ) # update the clips attribute for the db entry query = Event.select(Event.id).where( @@ -165,6 +159,7 @@ class EventCleanup(threading.Thread): Event.select( Event.id, Event.camera, + Event.thumbnail, ) .where( Event.camera == name, @@ -181,19 +176,12 @@ class EventCleanup(threading.Thread): # so no need to delete mp4 files for event in expired_events: events_to_update.append(event.id) + deleted = delete_event_images(event) - try: - media_name = f"{event.camera}-{event.id}" - media_path = Path( - f"{os.path.join(CLIPS_DIR, media_name)}.{file_extension}" + if not deleted: + logger.warning( + f"Unable to delete event images for {event.camera}: {event.id}" ) - media_path.unlink(missing_ok=True) - media_path = Path( - f"{os.path.join(CLIPS_DIR, media_name)}-clean.png" - ) - media_path.unlink(missing_ok=True) - except OSError as e: - logger.warning(f"Unable to delete event images: {e}") # update the clips attribute for the db entry for i in range(0, len(events_to_update), CHUNK_SIZE): diff --git a/frigate/events/external.py b/frigate/events/external.py index 0d3408975b..5423d08bea 100644 --- a/frigate/events/external.py +++ b/frigate/events/external.py @@ -1,6 +1,5 @@ """Handle external events created by the user.""" -import base64 import datetime import logging import os @@ -15,7 +14,7 @@ from numpy import ndarray from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum from frigate.comms.events_updater import EventUpdatePublisher from frigate.config import CameraConfig, FrigateConfig -from frigate.const import CLIPS_DIR +from frigate.const import CLIPS_DIR, THUMB_DIR from frigate.events.types import EventStateEnum, EventTypeEnum from frigate.util.image import draw_box_with_label @@ -55,9 +54,7 @@ class ExternalEventProcessor: rand_id = "".join(random.choices(string.ascii_lowercase + string.digits, k=6)) event_id = f"{now}-{rand_id}" - thumbnail = self._write_images( - camera_config, label, event_id, draw, snapshot_frame - ) + self._write_images(camera_config, label, event_id, draw, snapshot_frame) end = now + duration if duration is not None else None self.event_sender.publish( @@ -74,7 +71,6 @@ class ExternalEventProcessor: "camera": camera, "start_time": now - camera_config.record.event_pre_capture, "end_time": end, - "thumbnail": thumbnail, "has_clip": camera_config.record.enabled and include_recording, "has_snapshot": True, "type": source_type, @@ -134,9 +130,9 @@ class ExternalEventProcessor: event_id: str, draw: dict[str, any], img_frame: Optional[ndarray], - ) -> Optional[str]: + ) -> None: if img_frame is None: - return None + return # write clean snapshot if enabled if camera_config.snapshots.clean_copy: @@ -182,8 +178,9 @@ class ExternalEventProcessor: # create thumbnail with max height of 175 and save width = int(175 * img_frame.shape[1] / img_frame.shape[0]) thumb = cv2.resize(img_frame, dsize=(width, 175), interpolation=cv2.INTER_AREA) - ret, jpg = cv2.imencode(".jpg", thumb) - return base64.b64encode(jpg.tobytes()).decode("utf-8") + cv2.imwrite( + os.path.join(THUMB_DIR, camera_config.name, f"{event_id}.webp"), thumb + ) def stop(self): self.event_sender.stop() diff --git a/frigate/events/maintainer.py b/frigate/events/maintainer.py index fc02dd37ae..5cfa7c7167 100644 --- a/frigate/events/maintainer.py +++ b/frigate/events/maintainer.py @@ -23,7 +23,6 @@ def should_update_db(prev_event: Event, current_event: Event) -> bool: if ( prev_event["top_score"] != current_event["top_score"] or prev_event["entered_zones"] != current_event["entered_zones"] - or prev_event["thumbnail"] != current_event["thumbnail"] or prev_event["end_time"] != current_event["end_time"] or prev_event["average_estimated_speed"] != current_event["average_estimated_speed"] @@ -202,7 +201,7 @@ class EventProcessor(threading.Thread): Event.start_time: start_time, Event.end_time: end_time, Event.zones: list(event_data["entered_zones"]), - Event.thumbnail: event_data["thumbnail"], + Event.thumbnail: event_data.get("thumbnail"), Event.has_clip: event_data["has_clip"], Event.has_snapshot: event_data["has_snapshot"], Event.model_hash: first_detector.model.model_hash, @@ -258,7 +257,7 @@ class EventProcessor(threading.Thread): Event.camera: event_data["camera"], Event.start_time: event_data["start_time"], Event.end_time: event_data["end_time"], - Event.thumbnail: event_data["thumbnail"], + Event.thumbnail: event_data.get("thumbnail"), Event.has_clip: event_data["has_clip"], Event.has_snapshot: event_data["has_snapshot"], Event.zones: [], diff --git a/frigate/object_processing.py b/frigate/object_processing.py index 484f4a0824..aa966bab84 100644 --- a/frigate/object_processing.py +++ b/frigate/object_processing.py @@ -1,7 +1,6 @@ import datetime import json import logging -import os import queue import threading from collections import defaultdict @@ -16,13 +15,13 @@ from frigate.comms.dispatcher import Dispatcher from frigate.comms.events_updater import EventEndSubscriber, EventUpdatePublisher from frigate.comms.inter_process import InterProcessRequestor from frigate.config import ( + CameraMqttConfig, FrigateConfig, - MqttConfig, RecordConfig, SnapshotsConfig, ZoomingModeEnum, ) -from frigate.const import CLIPS_DIR, UPDATE_CAMERA_ACTIVITY +from frigate.const import UPDATE_CAMERA_ACTIVITY from frigate.events.types import EventStateEnum, EventTypeEnum from frigate.ptz.autotrack import PtzAutoTrackerThread from frigate.track.tracked_object import TrackedObject @@ -479,7 +478,7 @@ class TrackedObjectProcessor(threading.Thread): EventStateEnum.update, camera, frame_name, - obj.to_dict(include_thumbnail=True), + obj.to_dict(), ) ) @@ -491,41 +490,13 @@ class TrackedObjectProcessor(threading.Thread): obj.has_snapshot = self.should_save_snapshot(camera, obj) obj.has_clip = self.should_retain_recording(camera, obj) + # write thumbnail to disk if it will be saved as an event + if obj.has_snapshot or obj.has_clip: + obj.write_thumbnail_to_disk() + # write the snapshot to disk if obj.has_snapshot: - snapshot_config: SnapshotsConfig = self.config.cameras[camera].snapshots - jpg_bytes = obj.get_jpg_bytes( - timestamp=snapshot_config.timestamp, - bounding_box=snapshot_config.bounding_box, - crop=snapshot_config.crop, - height=snapshot_config.height, - quality=snapshot_config.quality, - ) - if jpg_bytes is None: - logger.warning(f"Unable to save snapshot for {obj.obj_data['id']}.") - else: - with open( - os.path.join(CLIPS_DIR, f"{camera}-{obj.obj_data['id']}.jpg"), - "wb", - ) as j: - j.write(jpg_bytes) - - # write clean snapshot if enabled - if snapshot_config.clean_copy: - png_bytes = obj.get_clean_png() - if png_bytes is None: - logger.warning( - f"Unable to save clean snapshot for {obj.obj_data['id']}." - ) - else: - with open( - os.path.join( - CLIPS_DIR, - f"{camera}-{obj.obj_data['id']}-clean.png", - ), - "wb", - ) as p: - p.write(png_bytes) + obj.write_snapshot_to_disk() if not obj.false_positive: message = { @@ -542,14 +513,15 @@ class TrackedObjectProcessor(threading.Thread): EventStateEnum.end, camera, frame_name, - obj.to_dict(include_thumbnail=True), + obj.to_dict(), ) ) def snapshot(camera, obj: TrackedObject, frame_name: str): - mqtt_config: MqttConfig = self.config.cameras[camera].mqtt + mqtt_config: CameraMqttConfig = self.config.cameras[camera].mqtt if mqtt_config.enabled and self.should_mqtt_snapshot(camera, obj): - jpg_bytes = obj.get_jpg_bytes( + jpg_bytes = obj.get_img_bytes( + ext="jpg", timestamp=mqtt_config.timestamp, bounding_box=mqtt_config.bounding_box, crop=mqtt_config.crop, diff --git a/frigate/track/tracked_object.py b/frigate/track/tracked_object.py index 0e7464bc2a..f1eb293282 100644 --- a/frigate/track/tracked_object.py +++ b/frigate/track/tracked_object.py @@ -1,8 +1,8 @@ """Object attribute.""" -import base64 import logging import math +import os from collections import defaultdict from statistics import median from typing import Optional @@ -13,8 +13,10 @@ import numpy as np from frigate.config import ( CameraConfig, ModelConfig, + SnapshotsConfig, UIConfig, ) +from frigate.const import CLIPS_DIR, THUMB_DIR from frigate.review.types import SeverityEnum from frigate.util.image import ( area, @@ -330,7 +332,7 @@ class TrackedObject: self.current_zones = current_zones return (thumb_update, significant_change, autotracker_update) - def to_dict(self, include_thumbnail: bool = False): + def to_dict(self): event = { "id": self.obj_data["id"], "camera": self.camera_config.name, @@ -365,9 +367,6 @@ class TrackedObject: "path_data": self.path_data, } - if include_thumbnail: - event["thumbnail"] = base64.b64encode(self.get_thumbnail()).decode("utf-8") - return event def is_active(self): @@ -379,22 +378,16 @@ class TrackedObject: > self.camera_config.detect.stationary.threshold ) - def get_thumbnail(self): - if ( - self.thumbnail_data is None - or self.thumbnail_data["frame_time"] not in self.frame_cache - ): - ret, jpg = cv2.imencode(".jpg", np.zeros((175, 175, 3), np.uint8)) - - jpg_bytes = self.get_jpg_bytes( - timestamp=False, bounding_box=False, crop=True, height=175 + def get_thumbnail(self, ext: str): + img_bytes = self.get_img_bytes( + ext, timestamp=False, bounding_box=False, crop=True, height=175 ) - if jpg_bytes: - return jpg_bytes + if img_bytes: + return img_bytes else: - ret, jpg = cv2.imencode(".jpg", np.zeros((175, 175, 3), np.uint8)) - return jpg.tobytes() + _, img = cv2.imencode(f".{ext}", np.zeros((175, 175, 3), np.uint8)) + return img.tobytes() def get_clean_png(self): if self.thumbnail_data is None: @@ -417,8 +410,14 @@ class TrackedObject: else: return None - def get_jpg_bytes( - self, timestamp=False, bounding_box=False, crop=False, height=None, quality=70 + def get_img_bytes( + self, + ext: str, + timestamp=False, + bounding_box=False, + crop=False, + height: int | None = None, + quality: int | None = None, ): if self.thumbnail_data is None: return None @@ -503,14 +502,69 @@ class TrackedObject: position=self.camera_config.timestamp_style.position, ) - ret, jpg = cv2.imencode( - ".jpg", best_frame, [int(cv2.IMWRITE_JPEG_QUALITY), quality] - ) + quality_params = None + + if ext == "jpg": + quality_params = [int(cv2.IMWRITE_JPEG_QUALITY), quality or 70] + elif ext == "webp": + quality_params = [int(cv2.IMWRITE_WEBP_QUALITY), quality or 60] + + ret, jpg = cv2.imencode(f".{ext}", best_frame, quality_params) + if ret: return jpg.tobytes() else: return None + def write_snapshot_to_disk(self) -> None: + snapshot_config: SnapshotsConfig = self.camera_config.snapshots + jpg_bytes = self.get_img_bytes( + ext="jpg", + timestamp=snapshot_config.timestamp, + bounding_box=snapshot_config.bounding_box, + crop=snapshot_config.crop, + height=snapshot_config.height, + quality=snapshot_config.quality, + ) + if jpg_bytes is None: + logger.warning(f"Unable to save snapshot for {self.obj_data['id']}.") + else: + with open( + os.path.join( + CLIPS_DIR, f"{self.camera_config.name}-{self.obj_data['id']}.jpg" + ), + "wb", + ) as j: + j.write(jpg_bytes) + + # write clean snapshot if enabled + if snapshot_config.clean_copy: + png_bytes = self.get_clean_png() + if png_bytes is None: + logger.warning( + f"Unable to save clean snapshot for {self.obj_data['id']}." + ) + else: + with open( + os.path.join( + CLIPS_DIR, + f"{self.camera_config.name}-{self.obj_data['id']}-clean.png", + ), + "wb", + ) as p: + p.write(png_bytes) + + def write_thumbnail_to_disk(self) -> None: + directory = os.path.join(THUMB_DIR, self.camera_config.name) + + if not os.path.exists(directory): + os.makedirs(directory) + + thumb_bytes = self.get_thumbnail("webp") + + with open(os.path.join(directory, f"{self.obj_data['id']}.webp"), "wb") as f: + f.write(thumb_bytes) + def zone_filtered(obj: TrackedObject, object_config): object_name = obj.obj_data["label"] diff --git a/frigate/util/path.py b/frigate/util/path.py new file mode 100644 index 0000000000..dbe51abe52 --- /dev/null +++ b/frigate/util/path.py @@ -0,0 +1,51 @@ +"""Path utilities.""" + +import base64 +import os +from pathlib import Path + +from frigate.const import CLIPS_DIR, THUMB_DIR +from frigate.models import Event + + +def get_event_thumbnail_bytes(event: Event) -> bytes | None: + if event.thumbnail: + return base64.b64decode(event.thumbnail) + else: + try: + with open( + os.path.join(THUMB_DIR, event.camera, f"{event.id}.webp"), "rb" + ) as f: + return f.read() + except Exception: + return None + + +### Deletion + + +def delete_event_images(event: Event) -> bool: + return delete_event_snapshot(event) and delete_event_thumbnail(event) + + +def delete_event_snapshot(event: Event) -> bool: + media_name = f"{event.camera}-{event.id}" + media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}.jpg") + + try: + media_path.unlink(missing_ok=True) + media_path = Path(f"{os.path.join(CLIPS_DIR, media_name)}-clean.png") + media_path.unlink(missing_ok=True) + return True + except OSError: + return False + + +def delete_event_thumbnail(event: Event) -> bool: + if event.thumbnail: + return True + else: + Path(os.path.join(THUMB_DIR, event.camera, f"{event.id}.webp")).unlink( + missing_ok=True + ) + return True diff --git a/migrations/028_optional_event_thumbnail.py b/migrations/028_optional_event_thumbnail.py new file mode 100644 index 0000000000..3e36a28ccd --- /dev/null +++ b/migrations/028_optional_event_thumbnail.py @@ -0,0 +1,36 @@ +"""Peewee migrations -- 028_optional_event_thumbnail.py. + +Some examples (model - class or model name):: + + > Model = migrator.orm['model_name'] # Return model in current state by name + + > migrator.sql(sql) # Run custom SQL + > migrator.python(func, *args, **kwargs) # Run python code + > migrator.create_model(Model) # Create a model (could be used as decorator) + > migrator.remove_model(model, cascade=True) # Remove a model + > migrator.add_fields(model, **fields) # Add fields to a model + > migrator.change_fields(model, **fields) # Change fields + > migrator.remove_fields(model, *field_names, cascade=True) + > migrator.rename_field(model, old_field_name, new_field_name) + > migrator.rename_table(model, new_table_name) + > migrator.add_index(model, *col_names, unique=False) + > migrator.drop_index(model, *col_names) + > migrator.add_not_null(model, *field_names) + > migrator.drop_not_null(model, *field_names) + > migrator.add_default(model, field_name, default) + +""" + +import peewee as pw + +from frigate.models import Event + +SQL = pw.SQL + + +def migrate(migrator, database, fake=False, **kwargs): + migrator.drop_not_null(Event, "thumbnail") + + +def rollback(migrator, database, fake=False, **kwargs): + migrator.add_not_null(Event, "thumbnail") diff --git a/web/src/components/card/SearchThumbnail.tsx b/web/src/components/card/SearchThumbnail.tsx index b7dd64e792..ed98e86b4e 100644 --- a/web/src/components/card/SearchThumbnail.tsx +++ b/web/src/components/card/SearchThumbnail.tsx @@ -80,7 +80,7 @@ export default function SearchThumbnail({ : undefined } draggable={false} - src={`${apiHost}api/events/${searchResult.id}/thumbnail.jpg`} + src={`${apiHost}api/events/${searchResult.id}/thumbnail.webp`} loading={isSafari ? "eager" : "lazy"} onLoad={() => { onImgLoad(); diff --git a/web/src/components/overlay/detail/ReviewDetailDialog.tsx b/web/src/components/overlay/detail/ReviewDetailDialog.tsx index 8d2f13d895..76234193c6 100644 --- a/web/src/components/overlay/detail/ReviewDetailDialog.tsx +++ b/web/src/components/overlay/detail/ReviewDetailDialog.tsx @@ -385,7 +385,7 @@ function EventItem({ src={ event.has_snapshot ? `${apiHost}api/events/${event.id}/snapshot.jpg` - : `${apiHost}api/events/${event.id}/thumbnail.jpg` + : `${apiHost}api/events/${event.id}/thumbnail.webp` } /> {hovered && ( @@ -400,7 +400,7 @@ function EventItem({ href={ event.has_snapshot ? `${apiHost}api/events/${event.id}/snapshot.jpg` - : `${apiHost}api/events/${event.id}/thumbnail.jpg` + : `${apiHost}api/events/${event.id}/thumbnail.webp` } > diff --git a/web/src/components/overlay/detail/SearchDetailDialog.tsx b/web/src/components/overlay/detail/SearchDetailDialog.tsx index dd088ad83e..03054d8115 100644 --- a/web/src/components/overlay/detail/SearchDetailDialog.tsx +++ b/web/src/components/overlay/detail/SearchDetailDialog.tsx @@ -511,7 +511,7 @@ function ObjectDetailsTab({ : undefined } draggable={false} - src={`${apiHost}api/events/${search.id}/thumbnail.jpg`} + src={`${apiHost}api/events/${search.id}/thumbnail.webp`} /> {config?.semantic_search.enabled && search.data.type == "object" && (