mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-22 05:23:01 +03:00
Refactor processors and add LPR postprocessing (#16722)
* recordings data pub/sub * function to process recording stream frames * model runner * lpr model runner * refactor to mixin class and use model runner * separate out realtime and post processors * move model and mixin folders * basic postprocessor * clean up * docs * postprocessing logic * clean up * return none if recordings are disabled * run postprocessor handle_requests too * tweak expansion * add put endpoint * postprocessor tweaks with endpoint
This commit is contained in:
@@ -17,7 +17,7 @@ from frigate.config import FrigateConfig
|
||||
from frigate.const import CONFIG_DIR, FACE_DIR
|
||||
from frigate.data_processing.types import DataProcessorMetrics
|
||||
from frigate.db.sqlitevecq import SqliteVecQueueDatabase
|
||||
from frigate.models import Event
|
||||
from frigate.models import Event, Recordings
|
||||
from frigate.util.builtin import serialize
|
||||
from frigate.util.services import listen
|
||||
|
||||
@@ -55,7 +55,7 @@ def manage_embeddings(config: FrigateConfig, metrics: DataProcessorMetrics) -> N
|
||||
timeout=max(60, 10 * len([c for c in config.cameras.values() if c.enabled])),
|
||||
load_vec_extension=True,
|
||||
)
|
||||
models = [Event]
|
||||
models = [Event, Recordings]
|
||||
db.bind(models)
|
||||
|
||||
maintainer = EmbeddingMaintainer(
|
||||
@@ -234,3 +234,8 @@ class EmbeddingsContext:
|
||||
EmbeddingsRequestEnum.embed_description.value,
|
||||
{"id": event_id, "description": description},
|
||||
)
|
||||
|
||||
def reprocess_plate(self, event: dict[str, any]) -> dict[str, any]:
|
||||
return self.requestor.send_data(
|
||||
EmbeddingsRequestEnum.reprocess_plate.value, {"event": event}
|
||||
)
|
||||
|
||||
@@ -20,18 +20,29 @@ from frigate.comms.event_metadata_updater import (
|
||||
)
|
||||
from frigate.comms.events_updater import EventEndSubscriber, EventUpdateSubscriber
|
||||
from frigate.comms.inter_process import InterProcessRequestor
|
||||
from frigate.comms.recordings_updater import (
|
||||
RecordingsDataSubscriber,
|
||||
RecordingsDataTypeEnum,
|
||||
)
|
||||
from frigate.config import FrigateConfig
|
||||
from frigate.const import (
|
||||
CLIPS_DIR,
|
||||
UPDATE_EVENT_DESCRIPTION,
|
||||
)
|
||||
from frigate.data_processing.real_time.api import RealTimeProcessorApi
|
||||
from frigate.data_processing.real_time.bird_processor import BirdProcessor
|
||||
from frigate.data_processing.real_time.face_processor import FaceProcessor
|
||||
from frigate.data_processing.real_time.license_plate_processor import (
|
||||
LicensePlateProcessor,
|
||||
from frigate.data_processing.common.license_plate.model import (
|
||||
LicensePlateModelRunner,
|
||||
)
|
||||
from frigate.data_processing.types import DataProcessorMetrics
|
||||
from frigate.data_processing.post.api import PostProcessorApi
|
||||
from frigate.data_processing.post.license_plate import (
|
||||
LicensePlatePostProcessor,
|
||||
)
|
||||
from frigate.data_processing.real_time.api import RealTimeProcessorApi
|
||||
from frigate.data_processing.real_time.bird import BirdRealTimeProcessor
|
||||
from frigate.data_processing.real_time.face import FaceRealTimeProcessor
|
||||
from frigate.data_processing.real_time.license_plate import (
|
||||
LicensePlateRealTimeProcessor,
|
||||
)
|
||||
from frigate.data_processing.types import DataProcessorMetrics, PostProcessDataEnum
|
||||
from frigate.events.types import EventTypeEnum
|
||||
from frigate.genai import get_genai_client
|
||||
from frigate.models import Event
|
||||
@@ -66,40 +77,71 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
if config.semantic_search.reindex:
|
||||
self.embeddings.reindex()
|
||||
|
||||
# create communication for updating event descriptions
|
||||
self.requestor = InterProcessRequestor()
|
||||
|
||||
self.event_subscriber = EventUpdateSubscriber()
|
||||
self.event_end_subscriber = EventEndSubscriber()
|
||||
self.event_metadata_subscriber = EventMetadataSubscriber(
|
||||
EventMetadataTypeEnum.regenerate_description
|
||||
)
|
||||
self.recordings_subscriber = RecordingsDataSubscriber(
|
||||
RecordingsDataTypeEnum.recordings_available_through
|
||||
)
|
||||
self.embeddings_responder = EmbeddingsResponder()
|
||||
self.frame_manager = SharedMemoryFrameManager()
|
||||
self.processors: list[RealTimeProcessorApi] = []
|
||||
|
||||
self.detected_license_plates: dict[str, dict[str, any]] = {}
|
||||
|
||||
# model runners to share between realtime and post processors
|
||||
if self.config.lpr.enabled:
|
||||
lpr_model_runner = LicensePlateModelRunner(self.requestor)
|
||||
|
||||
# realtime processors
|
||||
self.realtime_processors: list[RealTimeProcessorApi] = []
|
||||
|
||||
if self.config.face_recognition.enabled:
|
||||
self.processors.append(FaceProcessor(self.config, metrics))
|
||||
self.realtime_processors.append(FaceRealTimeProcessor(self.config, metrics))
|
||||
|
||||
if self.config.classification.bird.enabled:
|
||||
self.processors.append(BirdProcessor(self.config, metrics))
|
||||
self.realtime_processors.append(BirdRealTimeProcessor(self.config, metrics))
|
||||
|
||||
if self.config.lpr.enabled:
|
||||
self.processors.append(LicensePlateProcessor(self.config, metrics))
|
||||
self.realtime_processors.append(
|
||||
LicensePlateRealTimeProcessor(
|
||||
self.config, metrics, lpr_model_runner, self.detected_license_plates
|
||||
)
|
||||
)
|
||||
|
||||
# post processors
|
||||
self.post_processors: list[PostProcessorApi] = []
|
||||
|
||||
if self.config.lpr.enabled:
|
||||
self.post_processors.append(
|
||||
LicensePlatePostProcessor(
|
||||
self.config, metrics, lpr_model_runner, self.detected_license_plates
|
||||
)
|
||||
)
|
||||
|
||||
# create communication for updating event descriptions
|
||||
self.requestor = InterProcessRequestor()
|
||||
self.stop_event = stop_event
|
||||
self.tracked_events: dict[str, list[any]] = {}
|
||||
self.genai_client = get_genai_client(config)
|
||||
|
||||
# recordings data
|
||||
self.recordings_available_through: dict[str, float] = {}
|
||||
|
||||
def run(self) -> None:
|
||||
"""Maintain a SQLite-vec database for semantic search."""
|
||||
while not self.stop_event.is_set():
|
||||
self._process_requests()
|
||||
self._process_updates()
|
||||
self._process_recordings_updates()
|
||||
self._process_finalized()
|
||||
self._process_event_metadata()
|
||||
|
||||
self.event_subscriber.stop()
|
||||
self.event_end_subscriber.stop()
|
||||
self.recordings_subscriber.stop()
|
||||
self.event_metadata_subscriber.stop()
|
||||
self.embeddings_responder.stop()
|
||||
self.requestor.stop()
|
||||
@@ -129,13 +171,15 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
pack=False,
|
||||
)
|
||||
else:
|
||||
for processor in self.processors:
|
||||
resp = processor.handle_request(topic, data)
|
||||
processors = [self.realtime_processors, self.post_processors]
|
||||
for processor_list in processors:
|
||||
for processor in processor_list:
|
||||
resp = processor.handle_request(topic, data)
|
||||
|
||||
if resp is not None:
|
||||
return resp
|
||||
except Exception as e:
|
||||
logger.error(f"Unable to handle embeddings request {e}")
|
||||
logger.error(f"Unable to handle embeddings request {e}", exc_info=True)
|
||||
|
||||
self.embeddings_responder.check_for_request(_handle_request)
|
||||
|
||||
@@ -154,7 +198,7 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
camera_config = self.config.cameras[camera]
|
||||
|
||||
# no need to process updated objects if face recognition, lpr, genai are disabled
|
||||
if not camera_config.genai.enabled and len(self.processors) == 0:
|
||||
if not camera_config.genai.enabled and len(self.realtime_processors) == 0:
|
||||
return
|
||||
|
||||
# Create our own thumbnail based on the bounding box and the frame time
|
||||
@@ -171,7 +215,7 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
)
|
||||
return
|
||||
|
||||
for processor in self.processors:
|
||||
for processor in self.realtime_processors:
|
||||
processor.process_frame(data, yuv_frame)
|
||||
|
||||
# no need to save our own thumbnails if genai is not enabled
|
||||
@@ -202,7 +246,32 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
event_id, camera, updated_db = ended
|
||||
camera_config = self.config.cameras[camera]
|
||||
|
||||
for processor in self.processors:
|
||||
# call any defined post processors
|
||||
for processor in self.post_processors:
|
||||
if isinstance(processor, LicensePlatePostProcessor):
|
||||
recordings_available = self.recordings_available_through.get(camera)
|
||||
if (
|
||||
recordings_available is not None
|
||||
and event_id in self.detected_license_plates
|
||||
):
|
||||
processor.process_data(
|
||||
{
|
||||
"event_id": event_id,
|
||||
"camera": camera,
|
||||
"recordings_available": self.recordings_available_through[
|
||||
camera
|
||||
],
|
||||
"obj_data": self.detected_license_plates[event_id][
|
||||
"obj_data"
|
||||
],
|
||||
},
|
||||
PostProcessDataEnum.recording,
|
||||
)
|
||||
else:
|
||||
processor.process_data(event_id, PostProcessDataEnum.event_id)
|
||||
|
||||
# expire in realtime processors
|
||||
for processor in self.realtime_processors:
|
||||
processor.expire_object(event_id)
|
||||
|
||||
if updated_db:
|
||||
@@ -315,6 +384,24 @@ class EmbeddingMaintainer(threading.Thread):
|
||||
if event_id in self.tracked_events:
|
||||
del self.tracked_events[event_id]
|
||||
|
||||
def _process_recordings_updates(self) -> None:
|
||||
"""Process recordings updates."""
|
||||
while True:
|
||||
recordings_data = self.recordings_subscriber.check_for_update(timeout=0.01)
|
||||
|
||||
if recordings_data == None:
|
||||
break
|
||||
|
||||
camera, recordings_available_through_timestamp = recordings_data
|
||||
|
||||
self.recordings_available_through[camera] = (
|
||||
recordings_available_through_timestamp
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
f"{camera} now has recordings available through {recordings_available_through_timestamp}"
|
||||
)
|
||||
|
||||
def _process_event_metadata(self):
|
||||
# Check for regenerate description requests
|
||||
(topic, event_id, source) = self.event_metadata_subscriber.check_for_update(
|
||||
|
||||
Reference in New Issue
Block a user