Move genai review frame extraction off the embeddings loop (#24959)
CI / AMD64 Build (push) Canceled after 0s
CI / AMD64 Smoke Test (push) Canceled after 0s
CI / ARM Build (push) Canceled after 0s
CI / Jetson Jetpack 6 (push) Canceled after 0s
CI / AMD64 Extra Build (push) Canceled after 0s
CI / ARM Extra Build (push) Canceled after 0s
CI / Synaptics Build (push) Canceled after 0s
CI / Assemble and push default build (push) Canceled after 0s

* move genai review frame extraction off the embeddings loop

- Run frame gathering for finished review items on a thread so per-frame ffmpeg calls don't stall the embeddings maintainer
- Share one path between live and regenerated review descriptions
- Stop padding review start and end times in place, which widened the stored review item
- Lock timestamp expiry in EventsPerSecond since update() and eps() now run on different threads

* serialize recording frame extraction for review descriptions
This commit is contained in:
Josh Hawkins
2026-11-13 06:45:29 -07:00
committed by GitHub
parent b2f5315e09
commit bb04e6d49d
3 changed files with 126 additions and 60 deletions
@@ -68,6 +68,9 @@ class ReviewDescriptionProcessor(PostProcessorApi):
self.review_desc_dps = EventsPerSecond()
self.review_desc_dps.start()
# one review's ffmpeg extraction at a time, as on the maintainer loop
self._frame_extraction_lock = threading.Lock()
def calculate_frame_count(
self,
camera: str,
@@ -179,53 +182,72 @@ class ReviewDescriptionProcessor(PostProcessorApi):
):
return
image_source = camera_config.review.genai.image_source
frame_mode = camera_config.review.genai.frame_mode
threading.Thread(
target=self._describe_finished_review,
name=f"review_description_{id}",
daemon=True,
args=(
camera_config,
final_data,
camera_config.review.genai.image_source,
),
).start()
if image_source == ImageSourceEnum.recordings:
buffer_extension = get_recording_buffer_extension(
final_data["end_time"] - final_data["start_time"]
)
final_data["start_time"] -= buffer_extension
final_data["end_time"] += buffer_extension
def _describe_finished_review(
self,
camera_config: CameraConfig,
final_data: dict[str, Any],
image_source: ImageSourceEnum,
preview_fallback: bool = True,
named_objects: list[tuple[str, str]] | None = None,
) -> None:
"""Gather frames for a finished review item and start its description.
Runs off the maintainer loop since recording frames shell out to ffmpeg
once per frame.
"""
camera = final_data["camera"]
id = final_data["id"]
genai_config = camera_config.review.genai
frames: list[tuple[bytes, float]] = []
if image_source == ImageSourceEnum.recordings:
buffer_extension = get_recording_buffer_extension(
final_data["end_time"] - final_data["start_time"]
)
with self._frame_extraction_lock:
frames = self.get_recording_frames(
camera,
final_data["start_time"],
final_data["end_time"],
final_data["start_time"] - buffer_extension,
final_data["end_time"] + buffer_extension,
height=480, # Use 480p for good balance between quality and token usage
frame_mode=frame_mode,
frame_mode=genai_config.frame_mode,
)
if not frames:
# Fallback to preview frames if no recordings available
logger.warning(
f"No recording frames found for {camera}, falling back to preview frames"
)
frames = self.get_preview_frames_as_bytes(
camera,
final_data["start_time"],
final_data["end_time"],
final_data["thumb_path"],
id,
camera_config.review.genai.debug_save_thumbnails,
frame_mode,
)
elif camera_config.review.genai.debug_save_thumbnails:
if frames:
if genai_config.debug_save_thumbnails:
self.save_debug_recording_frames(id, frames)
elif not preview_fallback:
logger.error("No recording frames are available for review item %s", id)
return
else:
# Use preview frames
frames = self.get_preview_frames_as_bytes(
logger.warning(
"No recording frames found for %s, falling back to preview frames",
camera,
final_data["start_time"],
final_data["end_time"],
final_data["thumb_path"],
id,
camera_config.review.genai.debug_save_thumbnails,
frame_mode,
)
self.start_analysis(camera_config, final_data, frames)
if not frames:
frames = self.get_preview_frames_as_bytes(
camera,
final_data["start_time"],
final_data["end_time"],
final_data["thumb_path"],
id,
genai_config.debug_save_thumbnails,
genai_config.frame_mode,
)
self.start_analysis(camera_config, final_data, frames, named_objects)
def handle_request(self, topic: str, request_data: dict[str, Any]) -> str | None:
if topic == EmbeddingsRequestEnum.regenerate_review_description.value:
@@ -419,31 +441,12 @@ class ReviewDescriptionProcessor(PostProcessorApi):
logger.error("Review item %s has not ended yet", review_id)
return
buffer_extension = get_recording_buffer_extension(
final_data["end_time"] - final_data["start_time"]
)
frames = self.get_recording_frames(
str(review.camera),
final_data["start_time"] - buffer_extension,
final_data["end_time"] + buffer_extension,
height=480,
frame_mode=camera_config.review.genai.frame_mode,
)
if not frames:
logger.error(
"No recording frames are available for review item %s", review_id
)
return
if camera_config.review.genai.debug_save_thumbnails:
self.save_debug_recording_frames(review_id, frames)
self.start_analysis(
self._describe_finished_review(
camera_config,
final_data,
frames,
get_named_objects(
ImageSourceEnum.recordings,
preview_fallback=False,
named_objects=get_named_objects(
final_data["data"].get("detections") or [],
self.config.all_attributes,
),