diff --git a/docker/tensorrt/Dockerfile.arm64 b/docker/tensorrt/Dockerfile.arm64
index 7a88a03a6..5d5d5d939 100644
--- a/docker/tensorrt/Dockerfile.arm64
+++ b/docker/tensorrt/Dockerfile.arm64
@@ -86,6 +86,9 @@ RUN apt-get -qq update \
libx264-163 libx265-199 libegl1 \
&& rm -rf /var/lib/apt/lists/*
+# Fixes "Error loading shared libs"
+RUN mkdir -p /etc/ld.so.conf.d && echo /usr/lib/ffmpeg/jetson/lib/ > /etc/ld.so.conf.d/ffmpeg.conf
+
COPY --from=trt-wheels /etc/TENSORRT_VER /etc/TENSORRT_VER
RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
--mount=type=bind,from=trt-model-wheels,source=/trt-model-wheels,target=/deps/trt-model-wheels \
diff --git a/frigate/api/auth.py b/frigate/api/auth.py
index 1752b19c9..c0ed94d5c 100644
--- a/frigate/api/auth.py
+++ b/frigate/api/auth.py
@@ -136,7 +136,7 @@ def get_jwt_secret() -> str:
logger.debug("Using jwt secret from .jwt_secret file in config directory.")
with open(jwt_secret_file) as f:
try:
- jwt_secret = f.readline()
+ jwt_secret = f.readline().strip()
except Exception:
logger.warning(
"Unable to read jwt token from .jwt_secret file in config directory. A new jwt token will be created at each startup."
@@ -259,17 +259,24 @@ def auth(request: Request):
# pass the user header value from the upstream proxy if a mapping is specified
# or use anonymous if none are specified
user_header = proxy_config.header_map.user
- role_header = proxy_config.header_map.get("role", "Remote-Role")
+ role_header = proxy_config.header_map.role
success_response.headers["remote-user"] = (
request.headers.get(user_header, default="anonymous")
if user_header
else "anonymous"
)
- success_response.headers["remote-role"] = (
+ role_header = proxy_config.header_map.role
+ role = (
request.headers.get(role_header, default="viewer")
if role_header
else "viewer"
)
+
+ # if comma-separated with "admin", use "admin", else "viewer"
+ success_response.headers["remote-role"] = (
+ "admin" if role and "admin" in role else "viewer"
+ )
+
return success_response
# now apply authentication
@@ -359,14 +366,8 @@ def auth(request: Request):
@router.get("/profile")
def profile(request: Request):
username = request.headers.get("remote-user", "anonymous")
- if username != "anonymous":
- try:
- user = User.get_by_id(username)
- role = getattr(user, "role", "viewer")
- except DoesNotExist:
- role = "viewer" # Fallback if user deleted
- else:
- role = None
+ role = request.headers.get("remote-role", "viewer")
+
return JSONResponse(content={"username": username, "role": role})
diff --git a/frigate/api/event.py b/frigate/api/event.py
index 100bdfd9e..b47fe23c5 100644
--- a/frigate/api/event.py
+++ b/frigate/api/event.py
@@ -40,6 +40,7 @@ from frigate.api.defs.response.event_response import (
)
from frigate.api.defs.response.generic_response import GenericResponse
from frigate.api.defs.tags import Tags
+from frigate.comms.event_metadata_updater import EventMetadataTypeEnum
from frigate.const import CLIPS_DIR
from frigate.embeddings import EmbeddingsContext
from frigate.events.external import ExternalEventProcessor
@@ -969,27 +970,16 @@ def set_sub_label(
try:
event: Event = Event.get(Event.id == event_id)
except DoesNotExist:
- if not body.camera:
- return JSONResponse(
- content=(
- {
- "success": False,
- "message": "Event "
- + event_id
- + " not found and camera is not provided.",
- }
- ),
- status_code=404,
- )
-
event = None
if request.app.detected_frames_processor:
- tracked_obj: TrackedObject = (
- request.app.detected_frames_processor.camera_states[
- event.camera if event else body.camera
- ].tracked_objects.get(event_id)
- )
+ tracked_obj: TrackedObject = None
+
+ for state in request.app.detected_frames_processor.camera_states.values():
+ tracked_obj = state.tracked_objects.get(event_id)
+
+ if tracked_obj is not None:
+ break
else:
tracked_obj = None
@@ -1008,23 +998,9 @@ def set_sub_label(
new_sub_label = None
new_score = None
- if tracked_obj:
- tracked_obj.obj_data["sub_label"] = (new_sub_label, new_score)
-
- # update timeline items
- Timeline.update(
- data=Timeline.data.update({"sub_label": (new_sub_label, new_score)})
- ).where(Timeline.source_id == event_id).execute()
-
- if event:
- event.sub_label = new_sub_label
- data = event.data
- if new_sub_label is None:
- data["sub_label_score"] = None
- elif new_score is not None:
- data["sub_label_score"] = new_score
- event.data = data
- event.save()
+ request.app.event_metadata_updater.publish(
+ EventMetadataTypeEnum.sub_label, (event_id, new_sub_label, new_score)
+ )
return JSONResponse(
content={
@@ -1105,7 +1081,9 @@ def regenerate_description(
camera_config = request.app.frigate_config.cameras[event.camera]
if camera_config.genai.enabled:
- request.app.event_metadata_updater.publish((event.id, params.source))
+ request.app.event_metadata_updater.publish(
+ EventMetadataTypeEnum.regenerate_description, (event.id, params.source)
+ )
return JSONResponse(
content=(
diff --git a/frigate/app.py b/frigate/app.py
index cdb4877cc..af675eaaf 100644
--- a/frigate/app.py
+++ b/frigate/app.py
@@ -20,10 +20,7 @@ from frigate.camera import CameraMetrics, PTZMetrics
from frigate.comms.base_communicator import Communicator
from frigate.comms.config_updater import ConfigPublisher
from frigate.comms.dispatcher import Dispatcher
-from frigate.comms.event_metadata_updater import (
- EventMetadataPublisher,
- EventMetadataTypeEnum,
-)
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
from frigate.comms.inter_process import InterProcessCommunicator
from frigate.comms.mqtt import MqttClient
from frigate.comms.webpush import WebPushClient
@@ -327,9 +324,7 @@ class FrigateApp:
def init_inter_process_communicator(self) -> None:
self.inter_process_communicator = InterProcessCommunicator()
self.inter_config_updater = ConfigPublisher()
- self.event_metadata_updater = EventMetadataPublisher(
- EventMetadataTypeEnum.regenerate_description
- )
+ self.event_metadata_updater = EventMetadataPublisher()
self.inter_zmq_proxy = ZmqProxy()
def init_onvif(self) -> None:
@@ -600,6 +595,7 @@ class FrigateApp:
User.insert(
{
User.username: "admin",
+ User.role: "admin",
User.password_hash: password_hash,
User.notification_tokens: [],
}
diff --git a/frigate/comms/event_metadata_updater.py b/frigate/comms/event_metadata_updater.py
index 87e1889ce..f3301aef4 100644
--- a/frigate/comms/event_metadata_updater.py
+++ b/frigate/comms/event_metadata_updater.py
@@ -2,9 +2,6 @@
import logging
from enum import Enum
-from typing import Optional
-
-from frigate.events.types import RegenerateDescriptionEnum
from .zmq_proxy import Publisher, Subscriber
@@ -14,6 +11,7 @@ logger = logging.getLogger(__name__)
class EventMetadataTypeEnum(str, Enum):
all = ""
regenerate_description = "regenerate_description"
+ sub_label = "sub_label"
class EventMetadataPublisher(Publisher):
@@ -21,12 +19,11 @@ class EventMetadataPublisher(Publisher):
topic_base = "event_metadata/"
- def __init__(self, topic: EventMetadataTypeEnum) -> None:
- topic = topic.value
- super().__init__(topic)
+ def __init__(self) -> None:
+ super().__init__()
- def publish(self, payload: tuple[str, RegenerateDescriptionEnum]) -> None:
- super().publish(payload)
+ def publish(self, topic: EventMetadataTypeEnum, payload: any) -> None:
+ super().publish(payload, topic.value)
class EventMetadataSubscriber(Subscriber):
@@ -35,17 +32,14 @@ class EventMetadataSubscriber(Subscriber):
topic_base = "event_metadata/"
def __init__(self, topic: EventMetadataTypeEnum) -> None:
- topic = topic.value
- super().__init__(topic)
+ super().__init__(topic.value)
- def check_for_update(
- self, timeout: float = 1
- ) -> Optional[tuple[EventMetadataTypeEnum, str, RegenerateDescriptionEnum]]:
+ def check_for_update(self, timeout: float = 1) -> tuple | None:
return super().check_for_update(timeout)
- def _return_object(self, topic: str, payload: any) -> any:
+ def _return_object(self, topic: str, payload: tuple) -> tuple:
if payload is None:
- return (None, None, None)
+ return (None, None)
+
topic = EventMetadataTypeEnum[topic[len(self.topic_base) :]]
- event_id, source = payload
- return (topic, event_id, RegenerateDescriptionEnum(source))
+ return (topic, payload)
diff --git a/frigate/data_processing/common/license_plate/mixin.py b/frigate/data_processing/common/license_plate/mixin.py
index aa03bc985..c74949d9c 100644
--- a/frigate/data_processing/common/license_plate/mixin.py
+++ b/frigate/data_processing/common/license_plate/mixin.py
@@ -8,12 +8,11 @@ from typing import List, Optional, Tuple
import cv2
import numpy as np
-import requests
from Levenshtein import distance
from pyclipper import ET_CLOSEDPOLYGON, JT_ROUND, PyclipperOffset
from shapely.geometry import Polygon
-from frigate.const import FRIGATE_LOCALHOST
+from frigate.comms.event_metadata_updater import EventMetadataTypeEnum
from frigate.util.image import area
logger = logging.getLogger(__name__)
@@ -34,10 +33,10 @@ class LicensePlateProcessingMixin:
self.batch_size = 6
# Detection specific parameters
- self.min_size = 3
+ self.min_size = 8
self.max_size = 960
- self.box_thresh = 0.8
- self.mask_thresh = 0.8
+ self.box_thresh = 0.6
+ self.mask_thresh = 0.6
def _detect(self, image: np.ndarray) -> List[np.ndarray]:
"""
@@ -158,47 +157,40 @@ class LicensePlateProcessingMixin:
logger.debug("Model runners not loaded")
return [], [], []
- plate_points = self._detect(image)
- if len(plate_points) == 0:
- logger.debug("No points found by OCR detector model")
+ boxes = self._detect(image)
+ if len(boxes) == 0:
+ logger.debug("No boxes found by OCR detector model")
return [], [], []
- plate_points = self._sort_polygon(list(plate_points))
- plate_images = [self._crop_license_plate(image, x) for x in plate_points]
- rotated_images, _ = self._classify(plate_images)
+ boxes = self._sort_boxes(list(boxes))
+ plate_images = [self._crop_license_plate(image, x) for x in boxes]
- # debug rotated and classification result
if WRITE_DEBUG_IMAGES:
current_time = int(datetime.datetime.now().timestamp())
for i, img in enumerate(plate_images):
cv2.imwrite(
- f"debug/frames/license_plate_rotated_{current_time}_{i + 1}.jpg",
- img,
- )
- for i, img in enumerate(rotated_images):
- cv2.imwrite(
- f"debug/frames/license_plate_classified_{current_time}_{i + 1}.jpg",
+ f"debug/frames/license_plate_cropped_{current_time}_{i + 1}.jpg",
img,
)
# keep track of the index of each image for correct area calc later
- sorted_indices = np.argsort([x.shape[1] / x.shape[0] for x in rotated_images])
+ sorted_indices = np.argsort([x.shape[1] / x.shape[0] for x in plate_images])
reverse_mapping = {
idx: original_idx for original_idx, idx in enumerate(sorted_indices)
}
- results, confidences = self._recognize(rotated_images)
+ results, confidences = self._recognize(plate_images)
if results:
- license_plates = [""] * len(rotated_images)
- average_confidences = [[0.0]] * len(rotated_images)
- areas = [0] * len(rotated_images)
+ license_plates = [""] * len(plate_images)
+ average_confidences = [[0.0]] * len(plate_images)
+ areas = [0] * len(plate_images)
# map results back to original image order
for i, (plate, conf) in enumerate(zip(results, confidences)):
original_idx = reverse_mapping[i]
- height, width = rotated_images[original_idx].shape[:2]
+ height, width = plate_images[original_idx].shape[:2]
area = height * width
average_confidence = conf
@@ -206,7 +198,7 @@ class LicensePlateProcessingMixin:
# set to True to write each cropped image for debugging
if False:
save_image = cv2.cvtColor(
- rotated_images[original_idx], cv2.COLOR_RGB2BGR
+ plate_images[original_idx], cv2.COLOR_RGB2BGR
)
filename = f"debug/frames/plate_{original_idx}_{plate}_{area}.jpg"
cv2.imwrite(filename, save_image)
@@ -328,7 +320,7 @@ class LicensePlateProcessingMixin:
# Use pyclipper to shrink the polygon slightly based on the computed distance.
offset = PyclipperOffset()
offset.AddPath(points, JT_ROUND, ET_CLOSEDPOLYGON)
- points = np.array(offset.Execute(distance * 1.5)).reshape((-1, 1, 2))
+ points = np.array(offset.Execute(distance * 1.75)).reshape((-1, 1, 2))
# get the minimum bounding box around the shrunken polygon.
box, min_side = self._get_min_boxes(points)
@@ -453,46 +445,64 @@ class LicensePlateProcessingMixin:
)
@staticmethod
- def _clockwise_order(point: np.ndarray) -> np.ndarray:
+ def _clockwise_order(pts: np.ndarray) -> np.ndarray:
"""
- Arrange the points of a polygon in clockwise order based on their angular positions
- around the polygon's center.
+ Arrange the points of a polygon in order: top-left, top-right, bottom-right, bottom-left.
+ taken from https://github.com/PyImageSearch/imutils/blob/master/imutils/perspective.py
Args:
- point (np.ndarray): Array of points of the polygon.
+ pts (np.ndarray): Array of points of the polygon.
Returns:
- np.ndarray: Points ordered in clockwise direction.
+ np.ndarray: Points ordered clockwise starting from top-left.
"""
- center = point.mean(axis=0)
- return point[
- np.argsort(np.arctan2(point[:, 1] - center[1], point[:, 0] - center[0]))
- ]
+ # Sort the points based on their x-coordinates
+ x_sorted = pts[np.argsort(pts[:, 0]), :]
+
+ # Separate the left-most and right-most points
+ left_most = x_sorted[:2, :]
+ right_most = x_sorted[2:, :]
+
+ # Sort the left-most coordinates by y-coordinates
+ left_most = left_most[np.argsort(left_most[:, 1]), :]
+ (tl, bl) = left_most # Top-left and bottom-left
+
+ # Use the top-left as an anchor to calculate distances to right points
+ # The further point will be the bottom-right
+ distances = np.sqrt(
+ ((tl[0] - right_most[:, 0]) ** 2) + ((tl[1] - right_most[:, 1]) ** 2)
+ )
+
+ # Sort right points by distance (descending)
+ right_idx = np.argsort(distances)[::-1]
+ (br, tr) = right_most[right_idx, :] # Bottom-right and top-right
+
+ return np.array([tl, tr, br, bl])
@staticmethod
- def _sort_polygon(points):
+ def _sort_boxes(boxes):
"""
- Sort polygons based on their position in the image. If polygons are close in vertical
+ Sort polygons based on their position in the image. If boxes are close in vertical
position (within 5 pixels), sort them by horizontal position.
Args:
- points: List of polygons to sort.
+ points: detected text boxes with shape [4, 2]
Returns:
- List: Sorted list of polygons.
+ List: sorted boxes(array) with shape [4, 2]
"""
- points.sort(key=lambda x: (x[0][1], x[0][0]))
- for i in range(len(points) - 1):
+ boxes.sort(key=lambda x: (x[0][1], x[0][0]))
+ for i in range(len(boxes) - 1):
for j in range(i, -1, -1):
- if abs(points[j + 1][0][1] - points[j][0][1]) < 5 and (
- points[j + 1][0][0] < points[j][0][0]
+ if abs(boxes[j + 1][0][1] - boxes[j][0][1]) < 5 and (
+ boxes[j + 1][0][0] < boxes[j][0][0]
):
- temp = points[j]
- points[j] = points[j + 1]
- points[j + 1] = temp
+ temp = boxes[j]
+ boxes[j] = boxes[j + 1]
+ boxes[j + 1] = temp
else:
break
- return points
+ return boxes
@staticmethod
def _zero_pad(image: np.ndarray) -> np.ndarray:
@@ -583,9 +593,11 @@ class LicensePlateProcessingMixin:
for j in range(len(outputs)):
label, score = outputs[j]
results[indices[i + j]] = [label, score]
- # make sure we have high confidence if we need to flip a box, this will be rare in lpr
- if "180" in label and score >= 0.9:
- images[indices[i + j]] = cv2.rotate(images[indices[i + j]], 1)
+ # make sure we have high confidence if we need to flip a box
+ if "180" in label and score >= 0.7:
+ images[indices[i + j]] = cv2.rotate(
+ images[indices[i + j]], cv2.ROTATE_180
+ )
return images, results
@@ -682,7 +694,7 @@ class LicensePlateProcessingMixin:
)
height, width = image.shape[0:2]
if height * 1.0 / width >= 1.5:
- image = np.rot90(image, k=3)
+ image = cv2.rotate(image, cv2.ROTATE_90_CLOCKWISE)
return image
def _detect_license_plate(self, input: np.ndarray) -> tuple[int, int, int, int]:
@@ -942,9 +954,23 @@ class LicensePlateProcessingMixin:
return
license_plate_frame = cv2.cvtColor(frame, cv2.COLOR_YUV2BGR_I420)
+
+ # Expand the license_plate_box by 30%
+ box_array = np.array(license_plate_box)
+ expansion = (box_array[2:] - box_array[:2]) * 0.30
+ expanded_box = np.array(
+ [
+ license_plate_box[0] - expansion[0],
+ license_plate_box[1] - expansion[1],
+ license_plate_box[2] + expansion[0],
+ license_plate_box[3] + expansion[1],
+ ]
+ ).clip(0, [license_plate_frame.shape[1], license_plate_frame.shape[0]] * 2)
+
+ # Crop using the expanded box
license_plate_frame = license_plate_frame[
- license_plate_box[1] : license_plate_box[3],
- license_plate_box[0] : license_plate_box[2],
+ int(expanded_box[1]) : int(expanded_box[3]),
+ int(expanded_box[0]) : int(expanded_box[2]),
]
# double the size of the license plate frame for better OCR
@@ -1032,22 +1058,15 @@ class LicensePlateProcessingMixin:
)
# Send the result to the API
- resp = requests.post(
- f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
- json={
- "camera": obj_data.get("camera"),
- "subLabel": sub_label,
- "subLabelScore": avg_confidence,
- },
+ self.sub_label_publisher.publish(
+ EventMetadataTypeEnum.sub_label, (id, sub_label, avg_confidence)
)
-
- if resp.status_code == 200:
- self.detected_license_plates[id] = {
- "plate": top_plate,
- "char_confidences": top_char_confidences,
- "area": top_area,
- "obj_data": obj_data,
- }
+ self.detected_license_plates[id] = {
+ "plate": top_plate,
+ "char_confidences": top_char_confidences,
+ "area": top_area,
+ "obj_data": obj_data,
+ }
def handle_request(self, topic, request_data) -> dict[str, any] | None:
return
diff --git a/frigate/data_processing/post/license_plate.py b/frigate/data_processing/post/license_plate.py
index 2c80418c7..e5c8a29a8 100644
--- a/frigate/data_processing/post/license_plate.py
+++ b/frigate/data_processing/post/license_plate.py
@@ -8,6 +8,7 @@ import numpy as np
from peewee import DoesNotExist
from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
from frigate.config import FrigateConfig
from frigate.data_processing.common.license_plate.mixin import (
WRITE_DEBUG_IMAGES,
@@ -30,6 +31,7 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
def __init__(
self,
config: FrigateConfig,
+ sub_label_publisher: EventMetadataPublisher,
metrics: DataProcessorMetrics,
model_runner: LicensePlateModelRunner,
detected_license_plates: dict[str, dict[str, any]],
@@ -38,6 +40,7 @@ class LicensePlatePostProcessor(LicensePlateProcessingMixin, PostProcessorApi):
self.model_runner = model_runner
self.lpr_config = config.lpr
self.config = config
+ self.sub_label_publisher = sub_label_publisher
super().__init__(config, metrics, model_runner)
def process_data(
diff --git a/frigate/data_processing/real_time/bird.py b/frigate/data_processing/real_time/bird.py
index 01490d895..d942edf6f 100644
--- a/frigate/data_processing/real_time/bird.py
+++ b/frigate/data_processing/real_time/bird.py
@@ -5,10 +5,13 @@ import os
import cv2
import numpy as np
-import requests
+from frigate.comms.event_metadata_updater import (
+ EventMetadataPublisher,
+ EventMetadataTypeEnum,
+)
from frigate.config import FrigateConfig
-from frigate.const import FRIGATE_LOCALHOST, MODEL_CACHE_DIR
+from frigate.const import MODEL_CACHE_DIR
from frigate.util.object import calculate_region
from ..types import DataProcessorMetrics
@@ -23,9 +26,15 @@ logger = logging.getLogger(__name__)
class BirdRealTimeProcessor(RealTimeProcessorApi):
- def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics):
+ def __init__(
+ self,
+ config: FrigateConfig,
+ sub_label_publisher: EventMetadataPublisher,
+ metrics: DataProcessorMetrics,
+ ):
super().__init__(config, metrics)
self.interpreter: Interpreter = None
+ self.sub_label_publisher = sub_label_publisher
self.tensor_input_details: dict[str, any] = None
self.tensor_output_details: dict[str, any] = None
self.detected_birds: dict[str, float] = {}
@@ -134,17 +143,10 @@ class BirdRealTimeProcessor(RealTimeProcessorApi):
logger.debug(f"Score {score} is worse than previous score {previous_score}")
return
- resp = requests.post(
- f"{FRIGATE_LOCALHOST}/api/events/{obj_data['id']}/sub_label",
- json={
- "camera": obj_data.get("camera"),
- "subLabel": self.labelmap[best_id],
- "subLabelScore": score,
- },
+ self.sub_label_publisher.publish(
+ EventMetadataTypeEnum.sub_label, (id, self.labelmap[best_id], score)
)
-
- if resp.status_code == 200:
- self.detected_birds[obj_data["id"]] = score
+ self.detected_birds[obj_data["id"]] = score
def handle_request(self, topic, request_data):
return None
diff --git a/frigate/data_processing/real_time/face.py b/frigate/data_processing/real_time/face.py
index e7cf622e9..c88228651 100644
--- a/frigate/data_processing/real_time/face.py
+++ b/frigate/data_processing/real_time/face.py
@@ -11,11 +11,14 @@ from typing import Optional
import cv2
import numpy as np
-import requests
from frigate.comms.embeddings_updater import EmbeddingsRequestEnum
+from frigate.comms.event_metadata_updater import (
+ EventMetadataPublisher,
+ EventMetadataTypeEnum,
+)
from frigate.config import FrigateConfig
-from frigate.const import FACE_DIR, FRIGATE_LOCALHOST, MODEL_CACHE_DIR
+from frigate.const import FACE_DIR, MODEL_CACHE_DIR
from frigate.util.image import area
from ..types import DataProcessorMetrics
@@ -28,9 +31,15 @@ MIN_MATCHING_FACES = 2
class FaceRealTimeProcessor(RealTimeProcessorApi):
- def __init__(self, config: FrigateConfig, metrics: DataProcessorMetrics):
+ def __init__(
+ self,
+ config: FrigateConfig,
+ sub_label_publisher: EventMetadataPublisher,
+ metrics: DataProcessorMetrics,
+ ):
super().__init__(config, metrics)
self.face_config = config.face_recognition
+ self.sub_label_publisher = sub_label_publisher
self.face_detector: cv2.FaceDetectorYN = None
self.landmark_detector: cv2.face.FacemarkLBF = None
self.recognizer: cv2.face.LBPHFaceRecognizer = None
@@ -349,18 +358,10 @@ class FaceRealTimeProcessor(RealTimeProcessorApi):
self.__update_metrics(datetime.datetime.now().timestamp() - start)
return
- resp = requests.post(
- f"{FRIGATE_LOCALHOST}/api/events/{id}/sub_label",
- json={
- "camera": obj_data.get("camera"),
- "subLabel": sub_label,
- "subLabelScore": score,
- },
+ self.sub_label_publisher.publish(
+ EventMetadataTypeEnum.sub_label, (id, sub_label, score)
)
-
- if resp.status_code == 200:
- self.detected_faces[id] = face_score
-
+ self.detected_faces[id] = face_score
self.__update_metrics(datetime.datetime.now().timestamp() - start)
def handle_request(self, topic, request_data) -> dict[str, any] | None:
diff --git a/frigate/data_processing/real_time/license_plate.py b/frigate/data_processing/real_time/license_plate.py
index c8f0efa11..d2cb9f2a5 100644
--- a/frigate/data_processing/real_time/license_plate.py
+++ b/frigate/data_processing/real_time/license_plate.py
@@ -4,6 +4,7 @@ import logging
import numpy as np
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
from frigate.config import FrigateConfig
from frigate.data_processing.common.license_plate.mixin import (
LicensePlateProcessingMixin,
@@ -22,6 +23,7 @@ class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcess
def __init__(
self,
config: FrigateConfig,
+ sub_label_publisher: EventMetadataPublisher,
metrics: DataProcessorMetrics,
model_runner: LicensePlateModelRunner,
detected_license_plates: dict[str, dict[str, any]],
@@ -30,6 +32,7 @@ class LicensePlateRealTimeProcessor(LicensePlateProcessingMixin, RealTimeProcess
self.model_runner = model_runner
self.lpr_config = config.lpr
self.config = config
+ self.sub_label_publisher = sub_label_publisher
super().__init__(config, metrics)
def process_frame(self, obj_data: dict[str, any], frame: np.ndarray):
diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py
index b3bd6c204..2fa3eeb2c 100644
--- a/frigate/embeddings/maintainer.py
+++ b/frigate/embeddings/maintainer.py
@@ -15,6 +15,7 @@ from playhouse.sqliteq import SqliteQueueDatabase
from frigate.comms.embeddings_updater import EmbeddingsRequestEnum, EmbeddingsResponder
from frigate.comms.event_metadata_updater import (
+ EventMetadataPublisher,
EventMetadataSubscriber,
EventMetadataTypeEnum,
)
@@ -43,7 +44,7 @@ from frigate.data_processing.real_time.license_plate import (
LicensePlateRealTimeProcessor,
)
from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataEnum
-from frigate.events.types import EventTypeEnum
+from frigate.events.types import EventTypeEnum, RegenerateDescriptionEnum
from frigate.genai import get_genai_client
from frigate.models import Event
from frigate.types import TrackedObjectUpdateTypesEnum
@@ -89,6 +90,7 @@ class EmbeddingMaintainer(threading.Thread):
self.event_subscriber = EventUpdateSubscriber()
self.event_end_subscriber = EventEndSubscriber()
+ self.event_metadata_publisher = EventMetadataPublisher()
self.event_metadata_subscriber = EventMetadataSubscriber(
EventMetadataTypeEnum.regenerate_description
)
@@ -108,15 +110,27 @@ class EmbeddingMaintainer(threading.Thread):
self.realtime_processors: list[RealTimeProcessorApi] = []
if self.config.face_recognition.enabled:
- self.realtime_processors.append(FaceRealTimeProcessor(self.config, metrics))
+ self.realtime_processors.append(
+ FaceRealTimeProcessor(
+ self.config, self.event_metadata_publisher, metrics
+ )
+ )
if self.config.classification.bird.enabled:
- self.realtime_processors.append(BirdRealTimeProcessor(self.config, metrics))
+ self.realtime_processors.append(
+ BirdRealTimeProcessor(
+ self.config, self.event_metadata_publisher, metrics
+ )
+ )
if self.config.lpr.enabled:
self.realtime_processors.append(
LicensePlateRealTimeProcessor(
- self.config, metrics, lpr_model_runner, self.detected_license_plates
+ self.config,
+ self.event_metadata_publisher,
+ metrics,
+ lpr_model_runner,
+ self.detected_license_plates,
)
)
@@ -126,7 +140,11 @@ class EmbeddingMaintainer(threading.Thread):
if self.config.lpr.enabled:
self.post_processors.append(
LicensePlatePostProcessor(
- self.config, metrics, lpr_model_runner, self.detected_license_plates
+ self.config,
+ self.event_metadata_publisher,
+ metrics,
+ lpr_model_runner,
+ self.detected_license_plates,
)
)
@@ -150,6 +168,7 @@ class EmbeddingMaintainer(threading.Thread):
self.event_subscriber.stop()
self.event_end_subscriber.stop()
self.recordings_subscriber.stop()
+ self.event_metadata_publisher.stop()
self.event_metadata_subscriber.stop()
self.embeddings_responder.stop()
self.requestor.stop()
@@ -375,15 +394,17 @@ class EmbeddingMaintainer(threading.Thread):
def _process_event_metadata(self):
# Check for regenerate description requests
- (topic, event_id, source) = self.event_metadata_subscriber.check_for_update(
- timeout=0.01
- )
+ (topic, payload) = self.event_metadata_subscriber.check_for_update(timeout=0.01)
if topic is None:
return
+ event_id, source = payload
+
if event_id:
- self.handle_regenerate_description(event_id, source)
+ self.handle_regenerate_description(
+ event_id, RegenerateDescriptionEnum(source)
+ )
def _create_thumbnail(self, yuv_frame, box, height=500) -> Optional[bytes]:
"""Return jpg thumbnail of a region of the frame."""
diff --git a/frigate/events/audio.py b/frigate/events/audio.py
index 505802b8c..1a4fdd144 100644
--- a/frigate/events/audio.py
+++ b/frigate/events/audio.py
@@ -351,7 +351,8 @@ class AudioEventMaintainer(threading.Thread):
self.read_audio()
- stop_ffmpeg(self.audio_listener, self.logger)
+ if self.audio_listener:
+ stop_ffmpeg(self.audio_listener, self.logger)
self.logpipe.close()
self.requestor.stop()
self.config_subscriber.stop()
diff --git a/frigate/models.py b/frigate/models.py
index 26375432e..11b25b938 100644
--- a/frigate/models.py
+++ b/frigate/models.py
@@ -119,7 +119,7 @@ class User(Model): # type: ignore[misc]
username = CharField(null=False, primary_key=True, max_length=30)
role = CharField(
max_length=20,
- default="viewer",
+ default="admin",
)
password_hash = CharField(null=False, max_length=120)
notification_tokens = JSONField()
diff --git a/frigate/object_processing.py b/frigate/object_processing.py
index 8faf91cb5..d31ca83e1 100644
--- a/frigate/object_processing.py
+++ b/frigate/object_processing.py
@@ -9,10 +9,15 @@ from typing import Callable, Optional
import cv2
import numpy as np
+from peewee import DoesNotExist
from frigate.comms.config_updater import ConfigSubscriber
from frigate.comms.detections_updater import DetectionPublisher, DetectionTypeEnum
from frigate.comms.dispatcher import Dispatcher
+from frigate.comms.event_metadata_updater import (
+ EventMetadataSubscriber,
+ EventMetadataTypeEnum,
+)
from frigate.comms.events_updater import EventEndSubscriber, EventUpdatePublisher
from frigate.comms.inter_process import InterProcessRequestor
from frigate.config import (
@@ -24,6 +29,7 @@ from frigate.config import (
)
from frigate.const import UPDATE_CAMERA_ACTIVITY
from frigate.events.types import EventStateEnum, EventTypeEnum
+from frigate.models import Event, Timeline
from frigate.ptz.autotrack import PtzAutoTrackerThread
from frigate.track.tracked_object import TrackedObject
from frigate.util.image import (
@@ -446,6 +452,9 @@ class TrackedObjectProcessor(threading.Thread):
self.detection_publisher = DetectionPublisher(DetectionTypeEnum.video)
self.event_sender = EventUpdatePublisher()
self.event_end_subscriber = EventEndSubscriber()
+ self.sub_label_subscriber = EventMetadataSubscriber(
+ EventMetadataTypeEnum.sub_label
+ )
self.camera_activity: dict[str, dict[str, any]] = {}
@@ -684,6 +693,46 @@ class TrackedObjectProcessor(threading.Thread):
"""Returns the latest frame time for a given camera."""
return self.camera_states[camera].current_frame_time
+ def set_sub_label(
+ self, event_id: str, sub_label: str | None, score: float | None
+ ) -> None:
+ """Update sub label for given event id."""
+ tracked_obj: TrackedObject = None
+
+ for state in self.camera_states.values():
+ tracked_obj = state.tracked_objects.get(event_id)
+
+ if tracked_obj is not None:
+ break
+
+ try:
+ event: Event = Event.get(Event.id == event_id)
+ except DoesNotExist:
+ event = None
+
+ if not tracked_obj and not event:
+ return
+
+ if tracked_obj:
+ tracked_obj.obj_data["sub_label"] = (sub_label, score)
+
+ if event:
+ event.sub_label = sub_label
+ data = event.data
+ if sub_label is None:
+ data["sub_label_score"] = None
+ elif score is not None:
+ data["sub_label_score"] = score
+ event.data = data
+ event.save()
+
+ # update timeline items
+ Timeline.update(
+ data=Timeline.data.update({"sub_label": (sub_label, score)})
+ ).where(Timeline.source_id == event_id).execute()
+
+ return True
+
def force_end_all_events(self, camera: str, camera_state: CameraState):
"""Ends all active events on camera when disabling."""
last_frame_name = camera_state.previous_frame_id
@@ -741,6 +790,18 @@ class TrackedObjectProcessor(threading.Thread):
if not current_enabled:
continue
+ # check for sub label updates
+ while True:
+ (topic, payload) = self.sub_label_subscriber.check_for_update(
+ timeout=0.1
+ )
+
+ if not topic:
+ break
+
+ (event_id, sub_label, score) = payload
+ self.set_sub_label(event_id, sub_label, score)
+
try:
(
camera,
@@ -799,6 +860,7 @@ class TrackedObjectProcessor(threading.Thread):
self.detection_publisher.stop()
self.event_sender.stop()
self.event_end_subscriber.stop()
+ self.sub_label_subscriber.stop()
self.config_enabled_subscriber.stop()
logger.info("Exiting object processor...")
diff --git a/frigate/test/test_http.py b/frigate/test/test_http.py
index d6ff91a83..d23727672 100644
--- a/frigate/test/test_http.py
+++ b/frigate/test/test_http.py
@@ -2,6 +2,7 @@ import datetime
import logging
import os
import unittest
+from unittest.mock import Mock
from fastapi.testclient import TestClient
from peewee_migrate import Router
@@ -10,6 +11,7 @@ from playhouse.sqlite_ext import SqliteExtDatabase
from playhouse.sqliteq import SqliteQueueDatabase
from frigate.api.fastapi_app import create_fastapi_app
+from frigate.comms.event_metadata_updater import EventMetadataPublisher
from frigate.config import FrigateConfig
from frigate.const import BASE_DIR, CACHE_DIR
from frigate.models import Event, Recordings, Timeline
@@ -243,6 +245,7 @@ class TestHttp(unittest.TestCase):
assert len(events) == 1
def test_set_delete_sub_label(self):
+ mock_event_updater = Mock(spec=EventMetadataPublisher)
app = create_fastapi_app(
FrigateConfig(**self.minimal_config),
self.db,
@@ -252,11 +255,18 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
- None,
+ mock_event_updater,
)
id = "123456.random"
sub_label = "sub"
+ def update_event(topic, payload):
+ event = Event.get(id=id)
+ event.sub_label = payload[1]
+ event.save()
+
+ mock_event_updater.publish.side_effect = update_event
+
with TestClient(app) as client:
_insert_mock_event(id)
new_sub_label_response = client.post(
@@ -281,6 +291,7 @@ class TestHttp(unittest.TestCase):
assert event["sub_label"] == None
def test_sub_label_list(self):
+ mock_event_updater = Mock(spec=EventMetadataPublisher)
app = create_fastapi_app(
FrigateConfig(**self.minimal_config),
self.db,
@@ -290,11 +301,18 @@ class TestHttp(unittest.TestCase):
None,
None,
None,
- None,
+ mock_event_updater,
)
id = "123456.random"
sub_label = "sub"
+ def update_event(topic, payload):
+ event = Event.get(id=id)
+ event.sub_label = payload[1]
+ event.save()
+
+ mock_event_updater.publish.side_effect = update_event
+
with TestClient(app) as client:
_insert_mock_event(id)
client.post(
diff --git a/web/src/components/auth/AuthForm.tsx b/web/src/components/auth/AuthForm.tsx
index 617ce1693..85bd6bccb 100644
--- a/web/src/components/auth/AuthForm.tsx
+++ b/web/src/components/auth/AuthForm.tsx
@@ -87,7 +87,7 @@ export function UserAuthForm({ className, ...props }: UserAuthFormProps) {
return (
-
-
-
-
+ {selectedFaces?.length > 0 ? (
+
+
+
+ ) : (
+
+
+ {pageToggle != "train" && (
+
+ )}
+
+ )}
{pageToggle &&
(pageToggle == "train" ? (
@@ -227,6 +306,8 @@ export default function FaceLibrary() {
config={config}
attemptImages={trainImages}
faceNames={faces}
+ selectedFaces={selectedFaces}
+ onClickFace={onClickFace}
onRefresh={refreshFaces}
/>
) : (
@@ -244,22 +325,28 @@ type TrainingGridProps = {
config: FrigateConfig;
attemptImages: string[];
faceNames: string[];
+ selectedFaces: string[];
+ onClickFace: (image: string) => void;
onRefresh: () => void;
};
function TrainingGrid({
config,
attemptImages,
faceNames,
+ selectedFaces,
+ onClickFace,
onRefresh,
}: TrainingGridProps) {
return (
-
+
{attemptImages.map((image: string) => (
onClickFace(image)}
onRefresh={onRefresh}
/>
))}
@@ -271,12 +358,16 @@ type FaceAttemptProps = {
image: string;
faceNames: string[];
threshold: number;
+ selected: boolean;
+ onClick: () => void;
onRefresh: () => void;
};
function FaceAttempt({
image,
faceNames,
threshold,
+ selected,
+ onClick,
onRefresh,
}: FaceAttemptProps) {
const data = useMemo(() => {
@@ -336,30 +427,16 @@ function FaceAttempt({
});
}, [image, onRefresh]);
- const onDelete = useCallback(() => {
- axios
- .post(`/faces/train/delete`, { ids: [image] })
- .then((resp) => {
- if (resp.status == 200) {
- toast.success(`Successfully deleted face.`, {
- position: "top-center",
- });
- onRefresh();
- }
- })
- .catch((error) => {
- const errorMessage =
- error.response?.data?.message ||
- error.response?.data?.detail ||
- "Unknown error";
- toast.error(`Failed to delete: ${errorMessage}`, {
- position: "top-center",
- });
- });
- }, [image, onRefresh]);
-
return (
-
+
@@ -409,15 +486,6 @@ function FaceAttempt({
Reprocess Face
-
-
-
-
- Delete Face Attempt
-
diff --git a/web/src/views/events/EventView.tsx b/web/src/views/events/EventView.tsx
index b588fb1ff..f6b8af297 100644
--- a/web/src/views/events/EventView.tsx
+++ b/web/src/views/events/EventView.tsx
@@ -770,7 +770,12 @@ function DetectionReview({
/>
@@ -409,15 +486,6 @@ function FaceAttempt({