From 11ca1baddd9a3abf82240b5d2f5ac3aa6a069836 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Sat, 22 Nov 2025 11:51:27 -0700 Subject: [PATCH] Make object classification publish to tracked object update and add examples for state classification --- docs/docs/integrations/mqtt.md | 40 ++++++++++++++++++- .../real_time/custom_classification.py | 34 ++++++++++++++++ frigate/embeddings/maintainer.py | 2 + frigate/types.py | 1 + 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/docs/docs/integrations/mqtt.md b/docs/docs/integrations/mqtt.md index 1b4b54803..809c8c833 100644 --- a/docs/docs/integrations/mqtt.md +++ b/docs/docs/integrations/mqtt.md @@ -159,11 +159,44 @@ Message published for updates to tracked object metadata, for example: } ``` +#### Object Classification Update + +Message published when [object classification](/configuration/custom_classification/object_classification) reaches consensus on a classification result. + +**Sub label type:** + +```json +{ + "type": "classification", + "id": "1607123955.475377-mxklsc", + "camera": "front_door_cam", + "timestamp": 1607123958.748393, + "model": "person_classifier", + "sub_label": "delivery_person", + "score": 0.87 +} +``` + +**Attribute type:** + +```json +{ + "type": "classification", + "id": "1607123955.475377-mxklsc", + "camera": "front_door_cam", + "timestamp": 1607123958.748393, + "model": "helmet_detector", + "attribute": "yes", + "score": 0.92 +} +``` + ### `frigate/reviews` -Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. +Message published for each changed review item. The first message is published when the `detection` or `alert` is initiated. An `update` with the same ID will be published when: + - The severity changes from `detection` to `alert` - Additional objects are detected - An object is recognized via face, lpr, etc. @@ -308,6 +341,11 @@ Publishes transcribed text for audio detected on this camera. **NOTE:** Requires audio detection and transcription to be enabled +### `frigate//classification/` + +Publishes the current state detected by a state classification model for the camera. The topic name includes the model name as configured in your classification settings. +The published value is the detected state class name (e.g., `open`, `closed`, `on`, `off`). The state is only published when it changes, helping to reduce unnecessary MQTT traffic. + ### `frigate//enabled/set` Topic to turn Frigate's processing of a camera on and off. Expected values are `ON` and `OFF`. diff --git a/frigate/data_processing/real_time/custom_classification.py b/frigate/data_processing/real_time/custom_classification.py index 45ec30338..b5e8b1e35 100644 --- a/frigate/data_processing/real_time/custom_classification.py +++ b/frigate/data_processing/real_time/custom_classification.py @@ -1,6 +1,7 @@ """Real time processor that works with classification tflite models.""" import datetime +import json import logging import os from typing import Any @@ -21,6 +22,7 @@ from frigate.config.classification import ( ) from frigate.const import CLIPS_DIR, MODEL_CACHE_DIR from frigate.log import redirect_output_to_logger +from frigate.types import TrackedObjectUpdateTypesEnum from frigate.util.builtin import EventsPerSecond, InferenceSpeed, load_labels from frigate.util.object import box_overlaps, calculate_region @@ -284,6 +286,7 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): config: FrigateConfig, model_config: CustomClassificationConfig, sub_label_publisher: EventMetadataPublisher, + requestor: InterProcessRequestor, metrics: DataProcessorMetrics, ): super().__init__(config, metrics) @@ -292,6 +295,7 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): self.train_dir = os.path.join(CLIPS_DIR, self.model_config.name, "train") self.interpreter: Interpreter | None = None self.sub_label_publisher = sub_label_publisher + self.requestor = requestor self.tensor_input_details: dict[str, Any] | None = None self.tensor_output_details: dict[str, Any] | None = None self.classification_history: dict[str, list[tuple[str, float, float]]] = {} @@ -486,6 +490,8 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): ) if consensus_label is not None: + camera = obj_data["camera"] + if ( self.model_config.object_config.classification_type == ObjectClassificationType.sub_label @@ -494,6 +500,20 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): (object_id, consensus_label, consensus_score), EventMetadataTypeEnum.sub_label, ) + self.requestor.send_data( + "tracked_object_update", + json.dumps( + { + "type": TrackedObjectUpdateTypesEnum.classification, + "id": object_id, + "camera": camera, + "timestamp": now, + "model": self.model_config.name, + "sub_label": consensus_label, + "score": consensus_score, + } + ), + ) elif ( self.model_config.object_config.classification_type == ObjectClassificationType.attribute @@ -507,6 +527,20 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): ), EventMetadataTypeEnum.attribute.value, ) + self.requestor.send_data( + "tracked_object_update", + json.dumps( + { + "type": TrackedObjectUpdateTypesEnum.classification, + "id": object_id, + "camera": camera, + "timestamp": now, + "model": self.model_config.name, + "attribute": consensus_label, + "score": consensus_score, + } + ), + ) def handle_request(self, topic, request_data): if topic == EmbeddingsRequestEnum.reload_classification_model.value: diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 6f34f4556..78a251c42 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -195,6 +195,7 @@ class EmbeddingMaintainer(threading.Thread): self.config, model_config, self.event_metadata_publisher, + self.requestor, self.metrics, ) ) @@ -339,6 +340,7 @@ class EmbeddingMaintainer(threading.Thread): self.config, model_config, self.event_metadata_publisher, + self.requestor, self.metrics, ) diff --git a/frigate/types.py b/frigate/types.py index f342d27cd..6c5135616 100644 --- a/frigate/types.py +++ b/frigate/types.py @@ -30,3 +30,4 @@ class TrackedObjectUpdateTypesEnum(str, Enum): description = "description" face = "face" lpr = "lpr" + classification = "classification"