mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-10-14 18:52:47 +03:00
Josh HawkinsandGitHub
ff9eb8f184
* keep frame shm open and skip overwritten frames
Every reader closed and reopened a frame's shared memory segment on each frame, so each process paid a page fault on every page of a fresh mapping (about 760 for a 1080p frame) per frame. Readers now keep segments mapped. Capture stamps each slot's frame time in a per-camera `{camera}_frame_times` segment, and `get()` returns None when the slot no longer holds the requested frame, reopening once by name so a mapping left over from a same-size camera recreate heals itself. Capture also reuses a slot when the frame queue is full, since no reader was told about it.
* verify the frame for embeddings object updates
Tracked object events only carried the object's own frame_time, which stays at its last detection when the object isn't detected in a frame, so embeddings couldn't check the slot it read and could crop face or plate data from a frame that had already been overwritten. Events now carry the time of the frame they name, and embeddings passes it to `get()`.
* log when a frame slot was overwritten before it was read
A lapped read and a missing segment ended up as the same debug line at every caller, so there was no way to tell a slow reader from a missing frame.
* bring the FrameManager interface in line with its implementation
The abstract get() still had a timeout_ms signature from an older implementation and frame_times() wasn't declared, though capture and detect are typed against the interface. get() also didn't say how long the array it returns stays valid.
* drop a camera's cached frame mappings when it is removed or recreated
A reader that kept its mappings open stayed on a removed camera's segments, which pinned their shared memory until restart, and after a same-size recreate the ledger reopen only refreshed the slot being read while the other cached slots still returned the old camera's pixels. Readers now drop every cached slot for a camera on its remove update, and get() does the same when a ledger mismatch turns out to be a replaced ledger rather than a lapped slot.
* match a camera's frame names exactly in close_camera
The prefix match on `{camera}_frame` also caught another camera whose name starts with it, so removing `cam` dropped the cached ring of a camera named `cam_frame`.
630 lines
22 KiB
Python
630 lines
22 KiB
Python
"""Manages camera object detection processes."""
|
|
|
|
import logging
|
|
import queue
|
|
import threading
|
|
import time
|
|
from collections import deque
|
|
from datetime import UTC, datetime
|
|
from multiprocessing import Queue
|
|
from multiprocessing.synchronize import Event as MpEvent
|
|
from typing import Any
|
|
|
|
import cv2
|
|
|
|
from frigate.camera import CameraMetrics, PTZMetrics
|
|
from frigate.comms.inter_process import InterProcessRequestor
|
|
from frigate.config import CameraConfig, DetectConfig, LoggerConfig, ModelConfig
|
|
from frigate.config.camera.camera import CameraTypeEnum
|
|
from frigate.config.camera.updater import (
|
|
CameraConfigUpdateEnum,
|
|
CameraConfigUpdateSubscriber,
|
|
)
|
|
from frigate.const import (
|
|
PROCESS_PRIORITY_HIGH,
|
|
REPLAY_CAMERA_PREFIX,
|
|
REQUEST_REGION_GRID,
|
|
)
|
|
from frigate.motion import MotionDetector
|
|
from frigate.motion.improved_motion import ImprovedMotionDetector
|
|
from frigate.notices import raise_notice
|
|
from frigate.object_detection.base import RemoteObjectDetector
|
|
from frigate.ptz.autotrack import ptz_moving_at_frame_time
|
|
from frigate.track import ObjectTracker
|
|
from frigate.track.norfair_tracker import NorfairTracker
|
|
from frigate.track.tracked_object import TrackedObjectAttribute
|
|
from frigate.util.builtin import EventsPerSecond
|
|
from frigate.util.image import (
|
|
FrameManager,
|
|
SharedMemoryFrameManager,
|
|
draw_box_with_label,
|
|
)
|
|
from frigate.util.object import (
|
|
create_tensor_input,
|
|
get_cluster_candidates,
|
|
get_cluster_region,
|
|
get_cluster_region_from_grid,
|
|
get_min_region_size,
|
|
get_startup_regions,
|
|
inside_any,
|
|
intersects_any,
|
|
is_object_filtered,
|
|
reduce_detections,
|
|
)
|
|
from frigate.util.process import FrigateProcess
|
|
from frigate.util.time import get_tomorrow_at_time
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# this many frames dropped within the window because the shared detected frames
|
|
# queue was full means tracked object processing is falling behind
|
|
DROPPED_FRAMES_NOTICE_COUNT = 5
|
|
DROPPED_FRAMES_WINDOW_S = 30
|
|
|
|
DROPPED_FRAMES_NOTICE_INTERVAL_S = 60
|
|
|
|
|
|
class DroppedFrameTracker:
|
|
"""Raises a notice when a camera drops frames because object processing is behind."""
|
|
|
|
def __init__(self, camera: str) -> None:
|
|
# a debug replay can feed frames faster than real time
|
|
self._enabled = not camera.startswith(REPLAY_CAMERA_PREFIX)
|
|
self._drops: deque[float] = deque()
|
|
self._last_notice: float | None = None
|
|
|
|
# raising a notice waits on the main process, which answers every
|
|
# process's requests one at a time, so it runs off the frame loop
|
|
self._sender: threading.Thread | None = None
|
|
|
|
def dropped(self, now: float) -> None:
|
|
"""Record a dropped frame and raise the notice if enough were dropped.
|
|
|
|
Args:
|
|
now: Monotonic time of the drop in seconds
|
|
"""
|
|
if not self._enabled:
|
|
return
|
|
|
|
self._drops.append(now)
|
|
|
|
while now - self._drops[0] > DROPPED_FRAMES_WINDOW_S:
|
|
self._drops.popleft()
|
|
|
|
if (
|
|
len(self._drops) < DROPPED_FRAMES_NOTICE_COUNT
|
|
or (
|
|
self._last_notice is not None
|
|
and now - self._last_notice < DROPPED_FRAMES_NOTICE_INTERVAL_S
|
|
)
|
|
or (self._sender is not None and self._sender.is_alive())
|
|
):
|
|
return
|
|
|
|
self._drops.clear()
|
|
self._last_notice = now
|
|
self._sender = threading.Thread(
|
|
target=raise_notice,
|
|
args=("object_processing_behind",),
|
|
name="dropped_frames_notice",
|
|
daemon=True,
|
|
)
|
|
self._sender.start()
|
|
|
|
|
|
class CameraTracker(FrigateProcess):
|
|
def __init__(
|
|
self,
|
|
config: CameraConfig,
|
|
model_config: ModelConfig,
|
|
labelmap: dict[int, str],
|
|
detection_queue: Queue,
|
|
detected_objects_queue,
|
|
camera_metrics: CameraMetrics,
|
|
ptz_metrics: PTZMetrics,
|
|
region_grid: list[list[dict[str, Any]]],
|
|
stop_event: MpEvent,
|
|
log_config: LoggerConfig | None = None,
|
|
) -> None:
|
|
super().__init__(
|
|
stop_event,
|
|
PROCESS_PRIORITY_HIGH,
|
|
name=f"frigate.process:{config.name}",
|
|
daemon=True,
|
|
)
|
|
self.config = config
|
|
self.model_config = model_config
|
|
self.labelmap = labelmap
|
|
self.detection_queue = detection_queue
|
|
self.detected_objects_queue = detected_objects_queue
|
|
self.camera_metrics = camera_metrics
|
|
self.ptz_metrics = ptz_metrics
|
|
self.region_grid = region_grid
|
|
self.log_config = log_config
|
|
|
|
def run(self) -> None:
|
|
self.pre_run_setup(self.log_config)
|
|
|
|
# OpenCV only sees small images here, and its default per-core thread
|
|
# pool burns ~5x the CPU of running them inline
|
|
cv2.setNumThreads(1)
|
|
|
|
frame_queue = self.camera_metrics.frame_queue
|
|
frame_shape = self.config.frame_shape
|
|
|
|
motion_detector = ImprovedMotionDetector(
|
|
frame_shape,
|
|
self.config.motion,
|
|
self.config.detect.fps,
|
|
name=self.config.name,
|
|
ptz_metrics=self.ptz_metrics,
|
|
autotracking_enabled=self.config.onvif.autotracking.enabled,
|
|
)
|
|
object_detector = RemoteObjectDetector(
|
|
self.config.name,
|
|
self.labelmap,
|
|
self.detection_queue,
|
|
self.model_config,
|
|
self.stop_event,
|
|
)
|
|
|
|
object_tracker = NorfairTracker(self.config, self.ptz_metrics)
|
|
|
|
frame_manager = SharedMemoryFrameManager()
|
|
|
|
# create communication for region grid updates
|
|
requestor = InterProcessRequestor()
|
|
|
|
process_frames(
|
|
requestor,
|
|
frame_queue,
|
|
frame_shape,
|
|
self.model_config,
|
|
self.config,
|
|
frame_manager,
|
|
motion_detector,
|
|
object_detector,
|
|
object_tracker,
|
|
self.detected_objects_queue,
|
|
self.camera_metrics,
|
|
self.stop_event,
|
|
self.ptz_metrics,
|
|
self.region_grid,
|
|
)
|
|
|
|
# empty the frame queue
|
|
logger.info(f"{self.config.name}: emptying frame queue")
|
|
while not frame_queue.empty():
|
|
(frame_name, _) = frame_queue.get(False)
|
|
frame_manager.delete(frame_name)
|
|
|
|
logger.info(f"{self.config.name}: exiting subprocess")
|
|
|
|
|
|
def detect(
|
|
detect_config: DetectConfig,
|
|
object_detector,
|
|
frame,
|
|
model_config: ModelConfig,
|
|
region,
|
|
objects_to_track,
|
|
object_filters,
|
|
):
|
|
tensor_input = create_tensor_input(frame, model_config, region)
|
|
|
|
detections = []
|
|
region_detections = object_detector.detect(tensor_input)
|
|
for d in region_detections:
|
|
box = d[2]
|
|
size = region[2] - region[0]
|
|
x_min = int(max(0, (box[1] * size) + region[0]))
|
|
y_min = int(max(0, (box[0] * size) + region[1]))
|
|
x_max = int(min(detect_config.width - 1, (box[3] * size) + region[0]))
|
|
y_max = int(min(detect_config.height - 1, (box[2] * size) + region[1]))
|
|
|
|
# ignore objects that were detected outside the frame
|
|
if (x_min >= detect_config.width - 1) or (y_min >= detect_config.height - 1):
|
|
continue
|
|
|
|
width = x_max - x_min
|
|
height = y_max - y_min
|
|
area = width * height
|
|
ratio = width / max(1, height)
|
|
det = (d[0], d[1], (x_min, y_min, x_max, y_max), area, ratio, region)
|
|
# apply object filters
|
|
if is_object_filtered(det, objects_to_track, object_filters):
|
|
continue
|
|
detections.append(det)
|
|
return detections
|
|
|
|
|
|
def process_frames(
|
|
requestor: InterProcessRequestor,
|
|
frame_queue: Queue,
|
|
frame_shape: tuple[int, int],
|
|
model_config: ModelConfig,
|
|
camera_config: CameraConfig,
|
|
frame_manager: FrameManager,
|
|
motion_detector: MotionDetector,
|
|
object_detector: RemoteObjectDetector,
|
|
object_tracker: ObjectTracker,
|
|
detected_objects_queue: Queue,
|
|
camera_metrics: CameraMetrics,
|
|
stop_event: MpEvent,
|
|
ptz_metrics: PTZMetrics,
|
|
region_grid: list[list[dict[str, Any]]],
|
|
exit_on_empty: bool = False,
|
|
):
|
|
next_region_update = get_tomorrow_at_time(2)
|
|
config_subscriber = CameraConfigUpdateSubscriber(
|
|
None,
|
|
{camera_config.name: camera_config},
|
|
[
|
|
CameraConfigUpdateEnum.autotracking,
|
|
CameraConfigUpdateEnum.detect,
|
|
CameraConfigUpdateEnum.enabled,
|
|
CameraConfigUpdateEnum.motion,
|
|
CameraConfigUpdateEnum.objects,
|
|
CameraConfigUpdateEnum.onvif,
|
|
],
|
|
)
|
|
|
|
fps_tracker = EventsPerSecond()
|
|
fps_tracker.start()
|
|
dropped_frames = DroppedFrameTracker(camera_config.name)
|
|
|
|
startup_scan = True
|
|
stationary_frame_counter = 0
|
|
camera_enabled = True
|
|
|
|
region_min_size = get_min_region_size(model_config)
|
|
|
|
attributes_map = model_config.attributes_map
|
|
all_attributes = model_config.all_attributes
|
|
|
|
# remove license_plate from attributes if this camera is a dedicated LPR cam
|
|
if camera_config.type == CameraTypeEnum.lpr:
|
|
attributes_map = {
|
|
label: [attr for attr in attributes if attr != "license_plate"]
|
|
for label, attributes in model_config.attributes_map.items()
|
|
}
|
|
all_attributes = [
|
|
attr for attr in model_config.all_attributes if attr != "license_plate"
|
|
]
|
|
|
|
while not stop_event.is_set():
|
|
updated_configs = config_subscriber.check_for_updates()
|
|
|
|
if "enabled" in updated_configs:
|
|
prev_enabled = camera_enabled
|
|
camera_enabled = camera_config.enabled
|
|
|
|
if "motion" in updated_configs:
|
|
motion_detector.config = camera_config.motion
|
|
motion_detector.update_mask()
|
|
|
|
if "autotracking" in updated_configs or "onvif" in updated_configs:
|
|
motion_detector.autotracking_enabled = (
|
|
camera_config.onvif.autotracking.enabled
|
|
)
|
|
object_tracker.sync_trackers()
|
|
|
|
if (
|
|
not camera_enabled
|
|
and prev_enabled != camera_enabled
|
|
and camera_metrics.frame_queue.empty()
|
|
):
|
|
logger.debug(
|
|
f"Camera {camera_config.name} disabled, clearing tracked objects"
|
|
)
|
|
prev_enabled = camera_enabled
|
|
|
|
# Clear norfair's dictionaries
|
|
object_tracker.tracked_objects.clear()
|
|
object_tracker.disappeared.clear()
|
|
object_tracker.stationary_box_history.clear()
|
|
object_tracker.positions.clear()
|
|
object_tracker.track_id_map.clear()
|
|
|
|
# Clear internal norfair states
|
|
for trackers_by_type in object_tracker.trackers.values():
|
|
for tracker in trackers_by_type.values():
|
|
tracker.tracked_objects = []
|
|
for tracker in object_tracker.default_tracker.values():
|
|
tracker.tracked_objects = []
|
|
|
|
if not camera_enabled:
|
|
time.sleep(0.1)
|
|
continue
|
|
|
|
if datetime.now().astimezone(UTC) > next_region_update:
|
|
region_grid = requestor.send_data(REQUEST_REGION_GRID, camera_config.name)
|
|
next_region_update = get_tomorrow_at_time(2)
|
|
|
|
try:
|
|
if exit_on_empty:
|
|
frame_name, frame_time = frame_queue.get(False)
|
|
else:
|
|
frame_name, frame_time = frame_queue.get(True, 1)
|
|
except queue.Empty:
|
|
if exit_on_empty:
|
|
logger.info("Exiting track_objects...")
|
|
break
|
|
continue
|
|
|
|
camera_metrics.detection_frame.value = frame_time
|
|
ptz_metrics.frame_time.value = frame_time
|
|
|
|
frame = frame_manager.get(frame_name, (frame_shape[0] * 3 // 2, frame_shape[1]))
|
|
|
|
if frame is None:
|
|
logger.debug(
|
|
f"{camera_config.name}: frame {frame_time} is not in memory store."
|
|
)
|
|
continue
|
|
|
|
# look for motion if enabled
|
|
motion_boxes = motion_detector.detect(frame)
|
|
|
|
regions = []
|
|
consolidated_detections = []
|
|
|
|
# if detection is disabled
|
|
if not camera_config.detect.enabled:
|
|
object_tracker.match_and_update(frame_name, frame_time, [])
|
|
else:
|
|
# get stationary object ids
|
|
# check every Nth frame for stationary objects
|
|
# disappeared objects are not stationary
|
|
# also check for overlapping motion boxes
|
|
if stationary_frame_counter == camera_config.detect.stationary.interval:
|
|
stationary_frame_counter = 0
|
|
stationary_object_ids = []
|
|
else:
|
|
stationary_frame_counter += 1
|
|
stationary_object_ids = [
|
|
obj["id"]
|
|
for obj in object_tracker.tracked_objects.values()
|
|
# if it has exceeded the stationary threshold
|
|
if obj["motionless_count"]
|
|
>= camera_config.detect.stationary.threshold
|
|
# and it hasn't disappeared
|
|
and object_tracker.disappeared[obj["id"]] == 0
|
|
# and it doesn't overlap with any current motion boxes when not calibrating
|
|
and not intersects_any(
|
|
obj["box"],
|
|
[] if motion_detector.is_calibrating() else motion_boxes,
|
|
)
|
|
]
|
|
|
|
# get tracked object boxes that aren't stationary
|
|
tracked_object_boxes = [
|
|
(
|
|
# use existing object box for stationary objects
|
|
obj["estimate"]
|
|
if obj["motionless_count"]
|
|
< camera_config.detect.stationary.threshold
|
|
else obj["box"]
|
|
)
|
|
for obj in object_tracker.tracked_objects.values()
|
|
if obj["id"] not in stationary_object_ids
|
|
]
|
|
object_boxes = tracked_object_boxes + object_tracker.untracked_object_boxes
|
|
|
|
# get consolidated regions for tracked objects
|
|
regions = [
|
|
get_cluster_region(
|
|
frame_shape, region_min_size, candidate, object_boxes
|
|
)
|
|
for candidate in get_cluster_candidates(
|
|
frame_shape, region_min_size, object_boxes
|
|
)
|
|
]
|
|
|
|
# only add in the motion boxes when not calibrating and a ptz is not moving via autotracking
|
|
# the ptz timestamps are only maintained while autotracking is on, so gate
|
|
# on the config rather than trusting them to be reset otherwise
|
|
ptz_moving = camera_config.onvif.autotracking.enabled and (
|
|
ptz_moving_at_frame_time(
|
|
frame_time,
|
|
ptz_metrics.start_time.value,
|
|
ptz_metrics.stop_time.value,
|
|
)
|
|
)
|
|
|
|
if not motion_detector.is_calibrating() and not ptz_moving:
|
|
# find motion boxes that are not inside tracked object regions
|
|
standalone_motion_boxes = [
|
|
b for b in motion_boxes if not inside_any(b, regions)
|
|
]
|
|
|
|
if standalone_motion_boxes:
|
|
motion_clusters = get_cluster_candidates(
|
|
frame_shape,
|
|
region_min_size,
|
|
standalone_motion_boxes,
|
|
)
|
|
motion_regions = [
|
|
get_cluster_region_from_grid(
|
|
frame_shape,
|
|
region_min_size,
|
|
candidate,
|
|
standalone_motion_boxes,
|
|
region_grid,
|
|
)
|
|
for candidate in motion_clusters
|
|
]
|
|
regions += motion_regions
|
|
|
|
# if starting up, get the next startup scan region
|
|
if startup_scan:
|
|
for region in get_startup_regions(
|
|
frame_shape, region_min_size, region_grid
|
|
):
|
|
regions.append(region)
|
|
startup_scan = False
|
|
|
|
# resize regions and detect
|
|
# seed with stationary objects
|
|
detections = [
|
|
(
|
|
obj["label"],
|
|
obj["score"],
|
|
obj["box"],
|
|
obj["area"],
|
|
obj["ratio"],
|
|
obj["region"],
|
|
)
|
|
for obj in object_tracker.tracked_objects.values()
|
|
if obj["id"] in stationary_object_ids
|
|
]
|
|
|
|
for region in regions:
|
|
detections.extend(
|
|
detect(
|
|
camera_config.detect,
|
|
object_detector,
|
|
frame,
|
|
model_config,
|
|
region,
|
|
camera_config.objects.track,
|
|
camera_config.objects.filters,
|
|
)
|
|
)
|
|
|
|
consolidated_detections = reduce_detections(frame_shape, detections)
|
|
|
|
# if detection was run on this frame, consolidate
|
|
if len(regions) > 0:
|
|
tracked_detections = [
|
|
d for d in consolidated_detections if d[0] not in all_attributes
|
|
]
|
|
# now that we have refined our detections, we need to track objects
|
|
object_tracker.match_and_update(
|
|
frame_name, frame_time, tracked_detections
|
|
)
|
|
# else, just update the frame times for the stationary objects
|
|
else:
|
|
object_tracker.update_frame_times(frame_name, frame_time)
|
|
|
|
# build detections
|
|
detections = {}
|
|
for obj in object_tracker.tracked_objects.values():
|
|
detections[obj["id"]] = {**obj, "attributes": []}
|
|
|
|
# assign each detected attribute to the best matching object.
|
|
# iterate consolidated_detections once so attributes that appear under
|
|
# multiple parent labels in attributes_map (e.g. license_plate is in
|
|
# both "car" and "motorcycle") are not appended more than once
|
|
all_objects: list[dict[str, Any]] = object_tracker.tracked_objects.values()
|
|
detected_attributes = [
|
|
TrackedObjectAttribute(d)
|
|
for d in consolidated_detections
|
|
if d[0] in all_attributes
|
|
]
|
|
for attribute in detected_attributes:
|
|
filtered_objects = filter(
|
|
lambda o: attribute.label in attributes_map.get(o["label"], []),
|
|
all_objects,
|
|
)
|
|
selected_object_id = attribute.find_best_object(filtered_objects)
|
|
|
|
if selected_object_id is not None:
|
|
detections[selected_object_id]["attributes"].append(
|
|
attribute.get_tracking_data()
|
|
)
|
|
|
|
# debug object tracking
|
|
if False:
|
|
bgr_frame = cv2.cvtColor(
|
|
frame,
|
|
cv2.COLOR_YUV2BGR_I420,
|
|
)
|
|
object_tracker.debug_draw(bgr_frame, frame_time)
|
|
cv2.imwrite(
|
|
f"debug/frames/track-{'{:.6f}'.format(frame_time)}.jpg", bgr_frame
|
|
)
|
|
# debug
|
|
if False:
|
|
bgr_frame = cv2.cvtColor(
|
|
frame,
|
|
cv2.COLOR_YUV2BGR_I420,
|
|
)
|
|
|
|
for m_box in motion_boxes:
|
|
cv2.rectangle(
|
|
bgr_frame,
|
|
(m_box[0], m_box[1]),
|
|
(m_box[2], m_box[3]),
|
|
(0, 0, 255),
|
|
2,
|
|
)
|
|
|
|
for b in tracked_object_boxes:
|
|
cv2.rectangle(
|
|
bgr_frame,
|
|
(b[0], b[1]),
|
|
(b[2], b[3]),
|
|
(255, 0, 0),
|
|
2,
|
|
)
|
|
|
|
for obj in object_tracker.tracked_objects.values():
|
|
if obj["frame_time"] == frame_time:
|
|
thickness = 2
|
|
color = model_config.colormap.get(obj["label"], (255, 255, 255))
|
|
else:
|
|
thickness = 1
|
|
color = (255, 0, 0)
|
|
|
|
# draw the bounding boxes on the frame
|
|
box = obj["box"]
|
|
|
|
draw_box_with_label(
|
|
bgr_frame,
|
|
box[0],
|
|
box[1],
|
|
box[2],
|
|
box[3],
|
|
obj["label"],
|
|
obj["id"],
|
|
thickness=thickness,
|
|
color=color,
|
|
)
|
|
|
|
for region in regions:
|
|
cv2.rectangle(
|
|
bgr_frame,
|
|
(region[0], region[1]),
|
|
(region[2], region[3]),
|
|
(0, 255, 0),
|
|
2,
|
|
)
|
|
|
|
cv2.imwrite(
|
|
f"debug/frames/{camera_config.name}-{'{:.6f}'.format(frame_time)}.jpg",
|
|
bgr_frame,
|
|
)
|
|
# add to the queue if not full
|
|
if detected_objects_queue.full():
|
|
dropped_frames.dropped(time.monotonic())
|
|
continue
|
|
else:
|
|
fps_tracker.update()
|
|
camera_metrics.process_fps.value = fps_tracker.eps()
|
|
detected_objects_queue.put(
|
|
(
|
|
camera_config.name,
|
|
frame_name,
|
|
frame_time,
|
|
detections,
|
|
motion_boxes,
|
|
regions,
|
|
)
|
|
)
|
|
camera_metrics.detection_fps.value = object_detector.fps.eps()
|
|
|
|
motion_detector.stop()
|
|
requestor.stop()
|
|
config_subscriber.stop()
|