mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-22 05:23:01 +03:00
Move genai review frame extraction off the embeddings loop (#24959)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s
* move genai review frame extraction off the embeddings loop - Run frame gathering for finished review items on a thread so per-frame ffmpeg calls don't stall the embeddings maintainer - Share one path between live and regenerated review descriptions - Stop padding review start and end times in place, which widened the stored review item - Lock timestamp expiry in EventsPerSecond since update() and eps() now run on different threads * serialize recording frame extraction for review descriptions
This commit is contained in:
@@ -68,6 +68,9 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
self.review_desc_dps = EventsPerSecond()
|
||||
self.review_desc_dps.start()
|
||||
|
||||
# one review's ffmpeg extraction at a time, as on the maintainer loop
|
||||
self._frame_extraction_lock = threading.Lock()
|
||||
|
||||
def calculate_frame_count(
|
||||
self,
|
||||
camera: str,
|
||||
@@ -179,53 +182,72 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
):
|
||||
return
|
||||
|
||||
image_source = camera_config.review.genai.image_source
|
||||
frame_mode = camera_config.review.genai.frame_mode
|
||||
threading.Thread(
|
||||
target=self._describe_finished_review,
|
||||
name=f"review_description_{id}",
|
||||
daemon=True,
|
||||
args=(
|
||||
camera_config,
|
||||
final_data,
|
||||
camera_config.review.genai.image_source,
|
||||
),
|
||||
).start()
|
||||
|
||||
if image_source == ImageSourceEnum.recordings:
|
||||
buffer_extension = get_recording_buffer_extension(
|
||||
final_data["end_time"] - final_data["start_time"]
|
||||
)
|
||||
final_data["start_time"] -= buffer_extension
|
||||
final_data["end_time"] += buffer_extension
|
||||
def _describe_finished_review(
|
||||
self,
|
||||
camera_config: CameraConfig,
|
||||
final_data: dict[str, Any],
|
||||
image_source: ImageSourceEnum,
|
||||
preview_fallback: bool = True,
|
||||
named_objects: list[tuple[str, str]] | None = None,
|
||||
) -> None:
|
||||
"""Gather frames for a finished review item and start its description.
|
||||
|
||||
Runs off the maintainer loop since recording frames shell out to ffmpeg
|
||||
once per frame.
|
||||
"""
|
||||
camera = final_data["camera"]
|
||||
id = final_data["id"]
|
||||
genai_config = camera_config.review.genai
|
||||
frames: list[tuple[bytes, float]] = []
|
||||
|
||||
if image_source == ImageSourceEnum.recordings:
|
||||
buffer_extension = get_recording_buffer_extension(
|
||||
final_data["end_time"] - final_data["start_time"]
|
||||
)
|
||||
with self._frame_extraction_lock:
|
||||
frames = self.get_recording_frames(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
final_data["start_time"] - buffer_extension,
|
||||
final_data["end_time"] + buffer_extension,
|
||||
height=480, # Use 480p for good balance between quality and token usage
|
||||
frame_mode=frame_mode,
|
||||
frame_mode=genai_config.frame_mode,
|
||||
)
|
||||
|
||||
if not frames:
|
||||
# Fallback to preview frames if no recordings available
|
||||
logger.warning(
|
||||
f"No recording frames found for {camera}, falling back to preview frames"
|
||||
)
|
||||
frames = self.get_preview_frames_as_bytes(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
final_data["thumb_path"],
|
||||
id,
|
||||
camera_config.review.genai.debug_save_thumbnails,
|
||||
frame_mode,
|
||||
)
|
||||
elif camera_config.review.genai.debug_save_thumbnails:
|
||||
if frames:
|
||||
if genai_config.debug_save_thumbnails:
|
||||
self.save_debug_recording_frames(id, frames)
|
||||
elif not preview_fallback:
|
||||
logger.error("No recording frames are available for review item %s", id)
|
||||
return
|
||||
else:
|
||||
# Use preview frames
|
||||
frames = self.get_preview_frames_as_bytes(
|
||||
logger.warning(
|
||||
"No recording frames found for %s, falling back to preview frames",
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
final_data["thumb_path"],
|
||||
id,
|
||||
camera_config.review.genai.debug_save_thumbnails,
|
||||
frame_mode,
|
||||
)
|
||||
|
||||
self.start_analysis(camera_config, final_data, frames)
|
||||
if not frames:
|
||||
frames = self.get_preview_frames_as_bytes(
|
||||
camera,
|
||||
final_data["start_time"],
|
||||
final_data["end_time"],
|
||||
final_data["thumb_path"],
|
||||
id,
|
||||
genai_config.debug_save_thumbnails,
|
||||
genai_config.frame_mode,
|
||||
)
|
||||
|
||||
self.start_analysis(camera_config, final_data, frames, named_objects)
|
||||
|
||||
def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
|
||||
if topic == EmbeddingsRequestEnum.regenerate_review_description.value:
|
||||
@@ -419,31 +441,12 @@ class ReviewDescriptionProcessor(PostProcessorApi):
|
||||
logger.error("Review item %s has not ended yet", review_id)
|
||||
return
|
||||
|
||||
buffer_extension = get_recording_buffer_extension(
|
||||
final_data["end_time"] - final_data["start_time"]
|
||||
)
|
||||
frames = self.get_recording_frames(
|
||||
str(review.camera),
|
||||
final_data["start_time"] - buffer_extension,
|
||||
final_data["end_time"] + buffer_extension,
|
||||
height=480,
|
||||
frame_mode=camera_config.review.genai.frame_mode,
|
||||
)
|
||||
|
||||
if not frames:
|
||||
logger.error(
|
||||
"No recording frames are available for review item %s", review_id
|
||||
)
|
||||
return
|
||||
|
||||
if camera_config.review.genai.debug_save_thumbnails:
|
||||
self.save_debug_recording_frames(review_id, frames)
|
||||
|
||||
self.start_analysis(
|
||||
self._describe_finished_review(
|
||||
camera_config,
|
||||
final_data,
|
||||
frames,
|
||||
get_named_objects(
|
||||
ImageSourceEnum.recordings,
|
||||
preview_fallback=False,
|
||||
named_objects=get_named_objects(
|
||||
final_data["data"].get("detections") or [],
|
||||
self.config.all_attributes,
|
||||
),
|
||||
|
||||
Reference in New Issue
Block a user