mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-09-24 18:26:51 +03:00
Add options for review prompt style (#24166)
* Add script for testing genai review prompts * Add option for prompt styling * Add tests * Update docs
This commit is contained in:
@@ -28,3 +28,7 @@ core
|
||||
docs/src/components/DockerComposeGenerator/config/devices.ts
|
||||
docs/src/components/DockerComposeGenerator/config/hardware.ts
|
||||
docs/src/components/DockerComposeGenerator/config/ports.ts
|
||||
|
||||
# GenAI review prompt tester local data (frames from real cameras)
|
||||
testing-scripts/genai-review-examples/*
|
||||
!testing-scripts/genai-review-examples/README.md
|
||||
|
||||
@@ -496,6 +496,11 @@ review:
|
||||
- Animals in the garden
|
||||
# Optional: Preferred response language (default: English)
|
||||
preferred_language: English
|
||||
# Optional: Writing style preset for generated descriptions (default: shown below)
|
||||
# Options: "default", "natural", "concise", "detailed"
|
||||
# Presets adjust the tone and level of detail of the user-facing title,
|
||||
# summary, and scene description; "default" leaves the built-in prompt unchanged.
|
||||
response_style: default
|
||||
# Optional: Save thumbnails sent to the GenAI provider for review/debugging purposes (default: shown below)
|
||||
debug_save_thumbnails: False
|
||||
|
||||
|
||||
@@ -192,6 +192,39 @@ review:
|
||||
</TabItem>
|
||||
</ConfigTabs>
|
||||
|
||||
### Response Style
|
||||
|
||||
Different models respond to the built-in prompt with very different writing styles: some produce natural narration while others sound short and mechanical. The `response_style` option selects a writing style preset that rewords the prompt's instructions for the user-facing fields (the title, short summary, and scene description). Presets replace those instructions rather than adding extra ones, so the model never receives competing style directions.
|
||||
|
||||
Available presets:
|
||||
|
||||
- `default`: The built-in prompt, unchanged. This already reads like a neutral security report.
|
||||
- `natural`: Plain, everyday narration with flowing sentences and sentence-style headline titles. Useful when a model's output sounds robotic.
|
||||
- `concise`: As brief as possible while still covering each significant action, with terse two-to-four word titles.
|
||||
- `detailed`: Thorough descriptions and titles that include the most identifying specifics, like colors, clothing, and carried items.
|
||||
|
||||
Style presets only adjust how the user-facing text reads; the model's step-by-step observations and threat level scoring guidance are unaffected. Results vary by model, so it is worth comparing presets against saved debug output using `testing-scripts/genai_review_tester.py` in the Frigate repository.
|
||||
|
||||
<ConfigTabs>
|
||||
<TabItem value="ui">
|
||||
|
||||
Navigate to <NavPath path="Settings > Global configuration > Review" />.
|
||||
|
||||
- Set **GenAI config > Response style** to the desired preset (e.g., `natural`)
|
||||
|
||||
</TabItem>
|
||||
<TabItem value="yaml">
|
||||
|
||||
```yaml {4}
|
||||
review:
|
||||
genai:
|
||||
enabled: true
|
||||
response_style: natural
|
||||
```
|
||||
|
||||
</TabItem>
|
||||
</ConfigTabs>
|
||||
|
||||
## Review Reports
|
||||
|
||||
Along with individual review item summaries, Generative AI can also produce a single report of review items from all cameras marked "suspicious" over a specified time period (for example, a daily summary of suspicious activity while you're on vacation).
|
||||
|
||||
@@ -4,7 +4,13 @@ from pydantic import Field, field_validator
|
||||
|
||||
from ..base import FrigateBaseModel
|
||||
|
||||
__all__ = ["ReviewConfig", "DetectionsConfig", "AlertsConfig", "ImageSourceEnum"]
|
||||
__all__ = [
|
||||
"ReviewConfig",
|
||||
"DetectionsConfig",
|
||||
"AlertsConfig",
|
||||
"ImageSourceEnum",
|
||||
"ReviewResponseStyleEnum",
|
||||
]
|
||||
|
||||
|
||||
class ImageSourceEnum(str, Enum):
|
||||
@@ -14,6 +20,15 @@ class ImageSourceEnum(str, Enum):
|
||||
recordings = "recordings"
|
||||
|
||||
|
||||
class ReviewResponseStyleEnum(str, Enum):
|
||||
"""Writing style presets for GenAI review descriptions."""
|
||||
|
||||
default = "default"
|
||||
natural = "natural"
|
||||
concise = "concise"
|
||||
detailed = "detailed"
|
||||
|
||||
|
||||
DEFAULT_ALERT_OBJECTS = ["person", "car"]
|
||||
|
||||
|
||||
@@ -138,6 +153,11 @@ class GenAIReviewConfig(FrigateBaseModel):
|
||||
description="Preferred language to request from the GenAI provider for generated responses.",
|
||||
default=None,
|
||||
)
|
||||
response_style: ReviewResponseStyleEnum = Field(
|
||||
default=ReviewResponseStyleEnum.default,
|
||||
title="Response style",
|
||||
description="Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged.",
|
||||
)
|
||||
activity_context_prompt: str = Field(
|
||||
default="""### Normal Activity Indicators (Level 0)
|
||||
- Known/verified people in any zone at any time
|
||||
|
||||
@@ -601,6 +601,7 @@ def run_analysis(
|
||||
genai_config.preferred_language,
|
||||
genai_config.debug_save_thumbnails,
|
||||
genai_config.activity_context_prompt,
|
||||
genai_config.response_style,
|
||||
)
|
||||
review_inference_speed.update(datetime.datetime.now().timestamp() - start)
|
||||
|
||||
|
||||
@@ -104,6 +104,7 @@ class GenAIClient:
|
||||
preferred_language: str | None,
|
||||
debug_save: bool,
|
||||
activity_context_prompt: str,
|
||||
response_style: str = "default",
|
||||
) -> ReviewMetadata | None:
|
||||
"""Generate a description for the review item activity."""
|
||||
context_prompt = build_review_description_prompt(
|
||||
@@ -112,6 +113,7 @@ class GenAIClient:
|
||||
concerns,
|
||||
preferred_language,
|
||||
activity_context_prompt,
|
||||
response_style,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
|
||||
@@ -16,6 +16,48 @@ from frigate.config.ui import UnitSystemEnum
|
||||
from frigate.data_processing.post.types import ReviewMetadata
|
||||
from frigate.models import Event
|
||||
|
||||
# Base guidance per response field. `observations` is a reasoning scaffold,
|
||||
# not user-facing, so style presets never override it.
|
||||
REVIEW_DESCRIPTION_FIELD_GUIDELINES: dict[str, str] = {
|
||||
"observations": "Include the very start of the activity — for example, a vehicle entering the frame or pulling into the driveway — even if it lasts only a few frames and the rest of the clip is dominated by a longer activity. Include each arrival, departure, object handled, and notable change in position or state. Each item is a single concrete fact written as a complete sentence.",
|
||||
"scene": 'Describe how the sequence begins, then the progression of events — all significant movements and actions in order. For example, if a vehicle arrives and then a person exits, describe both sequentially. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name — do not replace them with generic terms. For unnamed objects (e.g., "person", "car"), refer to them naturally with articles (e.g., "a person", "the car"). Your description should align with and support the threat level you assign.',
|
||||
"title": "Name the primary activity across the observations, together with the location. An activity is what is being done with objects, tools, or surfaces; locomotion through the scene qualifies as the activity only when no other interaction is observed. For named subjects, always use their name. For unnamed objects, refer to them naturally with articles.",
|
||||
"shortSummary": "Briefly summarize the primary activity across the observations.",
|
||||
"potential_threat_level": "Must be consistent with your scene description and the activity patterns above.",
|
||||
}
|
||||
|
||||
# Style presets keyed by ReviewResponseStyleEnum value. Presets replace the
|
||||
# base guidance rather than append to it, so the prompt never carries
|
||||
# competing style instructions.
|
||||
REVIEW_RESPONSE_STYLES: dict[str, dict[str, str]] = {
|
||||
"natural": {
|
||||
"scene": 'Recount what happened the way a person would describe it to a neighbor, in plain everyday language: how the sequence begins, then each significant movement and action in the order it happens. Use flowing sentences that connect related actions, written the way people actually talk rather than like a surveillance report. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name rather than a generic term. Refer to unnamed objects naturally with articles. Stay factual, and keep the description consistent with the threat level you assign.',
|
||||
"title": 'Write the title as a short, sentence-case headline in present tense: the subject, then what they do, phrased the way you would text it to the homeowner. Describe only the action you see; do not assign the person a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. Name the main thing done, not just movement through the scene, unless movement is all that happens. For named subjects, always use their name.',
|
||||
"shortSummary": "Sum up the primary activity in one short, natural sentence, as if mentioning it to someone in passing.",
|
||||
},
|
||||
"concise": {
|
||||
"scene": 'Cover each significant movement and action in order using as few short, direct sentences as possible, omitting environmental and cosmetic detail unless it affects the assessment. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign.',
|
||||
"title": 'Write the title as a terse label of two to four words naming the specific activity observed and where it happened. Do not assign a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. For named subjects, always use their name.',
|
||||
"shortSummary": "Summarize the primary activity in one short sentence.",
|
||||
},
|
||||
"detailed": {
|
||||
"scene": 'Describe how the sequence begins, then the progression of events — all significant movements and actions in order, including the specifics that best identify the subjects: colors, clothing, carried items, positions, and paths of movement, plus environmental details like lighting changes when they stand out. Favor the most identifying details over exhaustive coverage, and keep every added detail observational rather than speculative. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name rather than a generic term. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign.',
|
||||
"title": 'Write the title as a specific description of who did what and where, in under roughly twelve words, including the most distinguishing visible detail of the subject, such as clothing or vehicle color. Do not assign a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. Name the main thing done, not just movement through the scene, unless movement is all that happens. For named subjects, always use their name.',
|
||||
"shortSummary": "Briefly summarize the primary activity across the observations, including the most identifying visible detail, such as vehicle color or clothing.",
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def get_review_field_guidelines(response_style: str = "default") -> dict[str, str]:
|
||||
"""Return per-field response guidance with the style preset applied.
|
||||
|
||||
"default" (or an unknown value) applies no overrides.
|
||||
"""
|
||||
return {
|
||||
**REVIEW_DESCRIPTION_FIELD_GUIDELINES,
|
||||
**REVIEW_RESPONSE_STYLES.get(response_style, {}),
|
||||
}
|
||||
|
||||
|
||||
def build_review_description_prompt(
|
||||
review_data: dict[str, Any],
|
||||
@@ -23,6 +65,7 @@ def build_review_description_prompt(
|
||||
concerns: list[str],
|
||||
preferred_language: str | None,
|
||||
activity_context_prompt: str,
|
||||
response_style: str = "default",
|
||||
) -> str:
|
||||
"""Build the prompt for review activity description generation."""
|
||||
|
||||
@@ -49,6 +92,8 @@ def build_review_description_prompt(
|
||||
else:
|
||||
return "\n- (No objects detected)"
|
||||
|
||||
fields = get_review_field_guidelines(response_style)
|
||||
|
||||
return f"""
|
||||
Your task is to analyze a sequence of images taken in chronological order from a security camera.
|
||||
|
||||
@@ -75,11 +120,11 @@ When forming your description:
|
||||
## Response Field Guidelines
|
||||
|
||||
Respond with a JSON object matching the provided schema. Field-specific guidance:
|
||||
- `observations`: Include the very start of the activity — for example, a vehicle entering the frame or pulling into the driveway — even if it lasts only a few frames and the rest of the clip is dominated by a longer activity. Include each arrival, departure, object handled, and notable change in position or state. Each item is a single concrete fact written as a complete sentence.
|
||||
- `scene`: Describe how the sequence begins, then the progression of events — all significant movements and actions in order. For example, if a vehicle arrives and then a person exits, describe both sequentially. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name — do not replace them with generic terms. For unnamed objects (e.g., "person", "car"), refer to them naturally with articles (e.g., "a person", "the car"). Your description should align with and support the threat level you assign.
|
||||
- `title`: Name the primary activity across the observations, together with the location. An activity is what is being done with objects, tools, or surfaces; locomotion through the scene qualifies as the activity only when no other interaction is observed. For named subjects, always use their name. For unnamed objects, refer to them naturally with articles.
|
||||
- `shortSummary`: Briefly summarize the primary activity across the observations.
|
||||
- `potential_threat_level`: Must be consistent with your scene description and the activity patterns above.
|
||||
- `observations`: {fields["observations"]}
|
||||
- `scene`: {fields["scene"]}
|
||||
- `title`: {fields["title"]}
|
||||
- `shortSummary`: {fields["shortSummary"]}
|
||||
- `potential_threat_level`: {fields["potential_threat_level"]}
|
||||
{get_concern_prompt()}
|
||||
|
||||
## Sequence Details
|
||||
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Tests for GenAI prompt builders."""
|
||||
|
||||
import unittest
|
||||
|
||||
from frigate.config.camera.review import ReviewResponseStyleEnum
|
||||
from frigate.genai.prompts import (
|
||||
REVIEW_DESCRIPTION_FIELD_GUIDELINES,
|
||||
REVIEW_RESPONSE_STYLES,
|
||||
build_review_description_prompt,
|
||||
get_review_field_guidelines,
|
||||
)
|
||||
|
||||
|
||||
class TestReviewResponseStyle(unittest.TestCase):
|
||||
def _build_prompt(self, response_style: str = "default") -> str:
|
||||
review_data = {
|
||||
"camera": "Front Door",
|
||||
"start": "Monday, 09:30 AM",
|
||||
"duration": 25,
|
||||
"zones": [],
|
||||
"unified_objects": ["person"],
|
||||
}
|
||||
return build_review_description_prompt(
|
||||
review_data,
|
||||
[b"fake-image"],
|
||||
[],
|
||||
None,
|
||||
"activity context",
|
||||
response_style,
|
||||
)
|
||||
|
||||
def test_default_style_leaves_prompt_unchanged(self):
|
||||
self.assertEqual(
|
||||
get_review_field_guidelines("default"),
|
||||
REVIEW_DESCRIPTION_FIELD_GUIDELINES,
|
||||
)
|
||||
self.assertEqual(self._build_prompt("default"), self._build_prompt())
|
||||
|
||||
def test_unknown_style_leaves_prompt_unchanged(self):
|
||||
self.assertEqual(self._build_prompt("unknown"), self._build_prompt())
|
||||
|
||||
def test_styles_replace_user_facing_field_guidance(self):
|
||||
default_prompt = self._build_prompt()
|
||||
for style, overrides in REVIEW_RESPONSE_STYLES.items():
|
||||
prompt = self._build_prompt(style)
|
||||
for field_name, guidance in overrides.items():
|
||||
self.assertIn(f"- `{field_name}`: {guidance}", prompt)
|
||||
self.assertNotIn(
|
||||
REVIEW_DESCRIPTION_FIELD_GUIDELINES[field_name], prompt
|
||||
)
|
||||
# Everything outside the overridden guidance lines is unchanged
|
||||
self.assertEqual(
|
||||
[
|
||||
line
|
||||
for line in prompt.splitlines()
|
||||
if not any(line.startswith(f"- `{f}`:") for f in overrides)
|
||||
],
|
||||
[
|
||||
line
|
||||
for line in default_prompt.splitlines()
|
||||
if not any(line.startswith(f"- `{f}`:") for f in overrides)
|
||||
],
|
||||
)
|
||||
|
||||
def test_styles_never_touch_reasoning_or_threat_fields(self):
|
||||
# observations is a reasoning scaffold and potential_threat_level is
|
||||
# scoring guidance; presets restyle only the user-facing fields.
|
||||
for overrides in REVIEW_RESPONSE_STYLES.values():
|
||||
self.assertTrue(
|
||||
set(overrides) <= {"scene", "title", "shortSummary"},
|
||||
f"unexpected override fields: {set(overrides)}",
|
||||
)
|
||||
|
||||
def test_config_enum_matches_style_presets(self):
|
||||
enum_styles = {e.value for e in ReviewResponseStyleEnum}
|
||||
self.assertEqual(enum_styles, {"default", *REVIEW_RESPONSE_STYLES})
|
||||
|
||||
def test_enum_value_selects_preset(self):
|
||||
prompt = self._build_prompt(ReviewResponseStyleEnum.natural)
|
||||
self.assertIn(
|
||||
REVIEW_RESPONSE_STYLES["natural"]["shortSummary"],
|
||||
prompt,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -0,0 +1,23 @@
|
||||
# GenAI Review Examples
|
||||
|
||||
Local example data for `testing-scripts/genai_review_tester.py`.
|
||||
|
||||
Each subfolder here should be a copy of a debug output folder saved by
|
||||
Frigate when `review.genai.debug_save_thumbnails: True` is enabled. Those
|
||||
folders are written to `clips/genai-requests/<review_id>/` and contain:
|
||||
|
||||
- Numbered frame images (`0.jpg`, `1.jpg`, ... or `.webp`) as sent to the
|
||||
GenAI provider
|
||||
- `prompt.txt` with the exact prompt Frigate built for the request
|
||||
- `response.txt` with the provider's response (not used by the tester)
|
||||
|
||||
Copy folders in, optionally rename them to something memorable (for example
|
||||
`driveway-night-delivery`), then run the tester from the repo root:
|
||||
|
||||
```bash
|
||||
python3 testing-scripts/genai_review_tester.py
|
||||
```
|
||||
|
||||
Everything in this folder except this README is gitignored, since the frames
|
||||
come from real cameras. Provider settings for the tester are stored here in
|
||||
`.settings.json`.
|
||||
@@ -0,0 +1,412 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Interactive tester for GenAI review description prompts.
|
||||
|
||||
Reuses Frigate's GenAI provider plugins and prompt builders so results match
|
||||
what Frigate produces at runtime, without needing a running Frigate instance.
|
||||
|
||||
Setup:
|
||||
1. Enable `review.genai.debug_save_thumbnails: True` in Frigate so debug
|
||||
output is saved under clips/genai-requests/<review_id>/.
|
||||
2. Copy one or more of those folders (numbered frame images plus
|
||||
prompt.txt) into testing-scripts/genai-review-examples/.
|
||||
3. Run from the repo root:
|
||||
python3 testing-scripts/genai_review_tester.py
|
||||
|
||||
The script presents a menu to edit provider settings (provider, base URL,
|
||||
optional API key, model) or run an example. Examples are selected with the
|
||||
up/down arrow keys and launched with Enter. A writing style preset can be
|
||||
applied on top of the saved prompt to compare tone between runs, and after a
|
||||
response the model can be asked follow-up questions about the same frames.
|
||||
|
||||
Provider settings are stored in
|
||||
testing-scripts/genai-review-examples/.settings.json (gitignored).
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import termios
|
||||
import time
|
||||
import tty
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
|
||||
from frigate.config.camera.genai import GenAIConfig, GenAIProviderEnum # noqa: E402
|
||||
from frigate.genai.prompts import ( # noqa: E402
|
||||
REVIEW_RESPONSE_STYLES,
|
||||
build_review_description_response_format,
|
||||
)
|
||||
|
||||
EXAMPLES_DIR = Path(__file__).resolve().parent / "genai-review-examples"
|
||||
SETTINGS_FILE = EXAMPLES_DIR / ".settings.json"
|
||||
IMAGE_EXTENSIONS = {".jpg", ".jpeg", ".png", ".webp"}
|
||||
|
||||
# Providers are imported individually so a missing SDK for an unused provider
|
||||
# does not break the script. Keys match GenAIProviderEnum values, values are
|
||||
# the module names under frigate.genai.plugins.
|
||||
PROVIDER_MODULES = {
|
||||
GenAIProviderEnum.openai: "openai",
|
||||
GenAIProviderEnum.azure_openai: "azure-openai",
|
||||
GenAIProviderEnum.gemini: "gemini",
|
||||
GenAIProviderEnum.ollama: "ollama",
|
||||
GenAIProviderEnum.llamacpp: "llama_cpp",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class TesterSettings:
|
||||
provider: str = ""
|
||||
base_url: str = ""
|
||||
api_key: str = ""
|
||||
model: str = ""
|
||||
timeout: int = 120
|
||||
runtime_options: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
@classmethod
|
||||
def load(cls) -> "TesterSettings":
|
||||
if SETTINGS_FILE.is_file():
|
||||
try:
|
||||
data = json.loads(SETTINGS_FILE.read_text())
|
||||
return cls(
|
||||
**{k: v for k, v in data.items() if k in cls.__annotations__}
|
||||
)
|
||||
except (json.JSONDecodeError, TypeError) as e:
|
||||
print(f"Ignoring invalid settings file: {e}")
|
||||
return cls()
|
||||
|
||||
def save(self) -> None:
|
||||
EXAMPLES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
SETTINGS_FILE.write_text(json.dumps(asdict(self), indent=2))
|
||||
|
||||
def describe(self) -> str:
|
||||
if not self.provider:
|
||||
return "not configured"
|
||||
key = "set" if self.api_key else "none"
|
||||
return (
|
||||
f"provider={self.provider} base_url={self.base_url or '(default)'} "
|
||||
f"model={self.model or '(none)'} api_key={key}"
|
||||
)
|
||||
|
||||
|
||||
def select_option(
|
||||
title: str, options: list[str], descriptions: list[str] | None = None
|
||||
) -> int | None:
|
||||
"""Render an arrow-key menu and return the selected index, or None.
|
||||
|
||||
Up/down (or j/k) moves, Enter selects, q or Esc cancels. Falls back to a
|
||||
numbered prompt when stdin is not a TTY.
|
||||
"""
|
||||
print(f"\n{title}")
|
||||
|
||||
if not sys.stdin.isatty():
|
||||
for i, option in enumerate(options):
|
||||
print(f" {i + 1}. {option}")
|
||||
choice = input("Selection (blank to cancel): ").strip()
|
||||
if not choice:
|
||||
return None
|
||||
try:
|
||||
index = int(choice) - 1
|
||||
except ValueError:
|
||||
return None
|
||||
return index if 0 <= index < len(options) else None
|
||||
|
||||
selected = 0
|
||||
|
||||
def render(first: bool) -> None:
|
||||
if not first:
|
||||
# Move the cursor back up and redraw in place
|
||||
sys.stdout.write(f"\x1b[{len(options)}A")
|
||||
for i, option in enumerate(options):
|
||||
marker = "❯" if i == selected else " "
|
||||
line = f" {marker} {option}"
|
||||
if descriptions and descriptions[i]:
|
||||
line += f" ({descriptions[i]})"
|
||||
sys.stdout.write(f"\x1b[2K{line}\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
render(True)
|
||||
fd = sys.stdin.fileno()
|
||||
old_attrs = termios.tcgetattr(fd)
|
||||
try:
|
||||
tty.setcbreak(fd)
|
||||
while True:
|
||||
ch = sys.stdin.read(1)
|
||||
if ch == "\x1b":
|
||||
seq = sys.stdin.read(1)
|
||||
if seq != "[":
|
||||
return None # bare Esc cancels
|
||||
arrow = sys.stdin.read(1)
|
||||
if arrow == "A":
|
||||
selected = (selected - 1) % len(options)
|
||||
elif arrow == "B":
|
||||
selected = (selected + 1) % len(options)
|
||||
elif ch in ("k",):
|
||||
selected = (selected - 1) % len(options)
|
||||
elif ch in ("j",):
|
||||
selected = (selected + 1) % len(options)
|
||||
elif ch in ("\r", "\n"):
|
||||
return selected
|
||||
elif ch in ("q", "\x03"):
|
||||
return None
|
||||
render(False)
|
||||
finally:
|
||||
termios.tcsetattr(fd, termios.TCSADRAIN, old_attrs)
|
||||
|
||||
|
||||
def prompt_text(label: str, current: str, secret: bool = False) -> str:
|
||||
"""Prompt for a text value, keeping the current value on empty input."""
|
||||
shown = ("***" if current else "") if secret else current
|
||||
value = input(f"{label} [{shown}]: ").strip()
|
||||
if not value:
|
||||
return current
|
||||
if value == "-":
|
||||
return ""
|
||||
return value
|
||||
|
||||
|
||||
def edit_settings(settings: TesterSettings) -> None:
|
||||
providers = [p.value for p in GenAIProviderEnum]
|
||||
index = select_option("Select a provider:", providers)
|
||||
if index is None:
|
||||
return
|
||||
|
||||
settings.provider = providers[index]
|
||||
print("Enter a value, press Enter to keep the current value, or '-' to clear.")
|
||||
settings.base_url = prompt_text("Base URL", settings.base_url)
|
||||
settings.api_key = prompt_text("API key (optional)", settings.api_key, secret=True)
|
||||
settings.model = prompt_text("Model", settings.model)
|
||||
settings.save()
|
||||
print(f"Saved settings: {settings.describe()}")
|
||||
|
||||
|
||||
def build_client(settings: TesterSettings) -> Any | None:
|
||||
"""Instantiate the Frigate provider client for the saved settings."""
|
||||
from frigate.genai import PROVIDERS
|
||||
|
||||
try:
|
||||
provider = GenAIProviderEnum(settings.provider)
|
||||
except ValueError:
|
||||
print("No valid provider configured. Edit settings first.")
|
||||
return None
|
||||
|
||||
module = PROVIDER_MODULES[provider]
|
||||
try:
|
||||
importlib.import_module(f"frigate.genai.plugins.{module}")
|
||||
except ImportError as e:
|
||||
print(f"Failed to import provider plugin '{module}': {e}")
|
||||
return None
|
||||
|
||||
config = GenAIConfig(
|
||||
provider=provider,
|
||||
base_url=settings.base_url or None,
|
||||
api_key=settings.api_key or None,
|
||||
model=settings.model,
|
||||
runtime_options=settings.runtime_options,
|
||||
)
|
||||
client = PROVIDERS[provider](config, timeout=settings.timeout, validate_model=False)
|
||||
if client.provider is None:
|
||||
print("Provider failed to initialize. Check the base URL and API key.")
|
||||
return None
|
||||
return client
|
||||
|
||||
|
||||
def list_examples() -> list[Path]:
|
||||
if not EXAMPLES_DIR.is_dir():
|
||||
return []
|
||||
return sorted(
|
||||
entry
|
||||
for entry in EXAMPLES_DIR.iterdir()
|
||||
if entry.is_dir() and not entry.name.startswith(".")
|
||||
)
|
||||
|
||||
|
||||
def load_frames(example: Path) -> list[bytes]:
|
||||
"""Load the example's frames as JPEG bytes in frame order.
|
||||
|
||||
Frames are saved by Frigate as <index>.jpg or <index>.webp. Non-JPEG
|
||||
images are re-encoded to JPEG to match what Frigate sends to providers.
|
||||
"""
|
||||
import cv2
|
||||
|
||||
def frame_order(path: Path) -> tuple[int, str]:
|
||||
try:
|
||||
return (int(path.stem), path.name)
|
||||
except ValueError:
|
||||
return (1 << 30, path.name)
|
||||
|
||||
frames: list[bytes] = []
|
||||
files = sorted(
|
||||
(f for f in example.iterdir() if f.suffix.lower() in IMAGE_EXTENSIONS),
|
||||
key=frame_order,
|
||||
)
|
||||
for file in files:
|
||||
if file.suffix.lower() in (".jpg", ".jpeg"):
|
||||
frames.append(file.read_bytes())
|
||||
continue
|
||||
|
||||
image = cv2.imread(str(file))
|
||||
if image is None:
|
||||
print(f" Skipping unreadable image {file.name}")
|
||||
continue
|
||||
ok, jpg = cv2.imencode(".jpg", image, [int(cv2.IMWRITE_JPEG_QUALITY), 100])
|
||||
if ok:
|
||||
frames.append(jpg.tobytes())
|
||||
return frames
|
||||
|
||||
|
||||
def apply_style(prompt: str, style: str) -> str:
|
||||
"""Apply a style preset to a saved prompt.
|
||||
|
||||
Presets replace the per-field response guidance lines, matching what
|
||||
Frigate builds at runtime. Handles both the current guidance format
|
||||
("- `scene`: ...") and the 0.17 format ("- `scene` (string): ...").
|
||||
"""
|
||||
overrides = REVIEW_RESPONSE_STYLES.get(style, {})
|
||||
for field_name, guidance in overrides.items():
|
||||
pattern = rf"^- `{field_name}`(?: \(string\))?: .*$"
|
||||
prompt, count = re.subn(
|
||||
pattern,
|
||||
lambda _: f"- `{field_name}`: {guidance}",
|
||||
prompt,
|
||||
count=1,
|
||||
flags=re.M,
|
||||
)
|
||||
if count == 0:
|
||||
print(f" Warning: no `{field_name}` guidance found in saved prompt")
|
||||
return prompt
|
||||
|
||||
|
||||
def pretty_print_response(response: str) -> None:
|
||||
try:
|
||||
parsed = json.loads(response)
|
||||
except json.JSONDecodeError:
|
||||
print(response)
|
||||
return
|
||||
print(json.dumps(parsed, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def followup_loop(client: Any, base_prompt: str, frames: list[bytes]) -> None:
|
||||
"""Let the user ask the model follow-up questions about the same frames."""
|
||||
transcript = base_prompt
|
||||
print("\nAsk follow-up questions about this sequence (blank line to finish).")
|
||||
while True:
|
||||
try:
|
||||
question = input("follow-up> ").strip()
|
||||
except EOFError:
|
||||
return
|
||||
if not question:
|
||||
return
|
||||
|
||||
prompt = (
|
||||
f"{transcript}\n\n---\n"
|
||||
"Answer the follow-up question below about the same sequence of "
|
||||
"frames. Respond in plain text, not JSON.\n"
|
||||
f"Question: {question}"
|
||||
)
|
||||
start = time.monotonic()
|
||||
answer = client._send(prompt, frames)
|
||||
elapsed = time.monotonic() - start
|
||||
if answer is None:
|
||||
print("No response from provider (see logs above).")
|
||||
continue
|
||||
print(f"\n{answer}\n({elapsed:.1f}s)")
|
||||
transcript = f"{prompt}\n\nYour answer:\n{answer}"
|
||||
|
||||
|
||||
def run_example(settings: TesterSettings) -> None:
|
||||
examples = list_examples()
|
||||
if not examples:
|
||||
print(
|
||||
f"No examples found. Copy debug output folders from "
|
||||
f"clips/genai-requests/ into {EXAMPLES_DIR}/"
|
||||
)
|
||||
return
|
||||
|
||||
index = select_option(
|
||||
"Select an example (arrow keys, Enter to launch):",
|
||||
[e.name for e in examples],
|
||||
)
|
||||
if index is None:
|
||||
return
|
||||
example = examples[index]
|
||||
|
||||
styles = ["default"] + list(REVIEW_RESPONSE_STYLES)
|
||||
style_index = select_option("Select a response style:", styles)
|
||||
if style_index is None:
|
||||
return
|
||||
style = styles[style_index]
|
||||
|
||||
prompt_file = example / "prompt.txt"
|
||||
if not prompt_file.is_file():
|
||||
print(f"{example.name} has no prompt.txt, cannot run")
|
||||
return
|
||||
prompt = apply_style(prompt_file.read_text(), style)
|
||||
|
||||
frames = load_frames(example)
|
||||
if not frames:
|
||||
print(f"{example.name} contains no frame images, cannot run")
|
||||
return
|
||||
|
||||
client = build_client(settings)
|
||||
if client is None:
|
||||
return
|
||||
|
||||
# Keep the other_concerns field in the schema only when the saved prompt
|
||||
# asked for it; the schema builder only checks truthiness of the list.
|
||||
concerns = ["_"] if "other_concerns" in prompt else []
|
||||
response_format = build_review_description_response_format(concerns)
|
||||
|
||||
print(
|
||||
f"\nSending {len(frames)} frames to {settings.provider} "
|
||||
f"(model={settings.model or '(none)'}, style={style})..."
|
||||
)
|
||||
start = time.monotonic()
|
||||
response = client._send(prompt, frames, response_format)
|
||||
elapsed = time.monotonic() - start
|
||||
|
||||
if response is None:
|
||||
print("No response from provider (see logs above).")
|
||||
return
|
||||
|
||||
print(f"\nResponse ({elapsed:.1f}s):\n")
|
||||
pretty_print_response(response)
|
||||
followup_loop(client, f"{prompt}\n\nYour analysis:\n{response}", frames)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
os.chdir(REPO_ROOT)
|
||||
logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
|
||||
EXAMPLES_DIR.mkdir(parents=True, exist_ok=True)
|
||||
settings = TesterSettings.load()
|
||||
|
||||
print("Frigate GenAI review prompt tester")
|
||||
while True:
|
||||
choice = select_option(
|
||||
f"Menu (settings: {settings.describe()}):",
|
||||
["Run a review example", "Edit provider settings", "Quit"],
|
||||
)
|
||||
if choice == 0:
|
||||
if not settings.provider:
|
||||
print("Configure provider settings first.")
|
||||
edit_settings(settings)
|
||||
if not settings.provider:
|
||||
continue
|
||||
run_example(settings)
|
||||
elif choice == 1:
|
||||
edit_settings(settings)
|
||||
else:
|
||||
return
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
try:
|
||||
main()
|
||||
except KeyboardInterrupt:
|
||||
print()
|
||||
@@ -657,6 +657,10 @@
|
||||
"label": "Preferred language",
|
||||
"description": "Preferred language to request from the GenAI provider for generated responses."
|
||||
},
|
||||
"response_style": {
|
||||
"label": "Response style",
|
||||
"description": "Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged."
|
||||
},
|
||||
"activity_context_prompt": {
|
||||
"label": "Activity context prompt",
|
||||
"description": "Custom prompt describing what is and is not suspicious activity to provide context for GenAI summaries."
|
||||
|
||||
@@ -1028,6 +1028,10 @@
|
||||
"label": "Preferred language",
|
||||
"description": "Preferred language to request from the GenAI provider for generated responses."
|
||||
},
|
||||
"response_style": {
|
||||
"label": "Response style",
|
||||
"description": "Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged."
|
||||
},
|
||||
"activity_context_prompt": {
|
||||
"label": "Activity context prompt",
|
||||
"description": "Custom prompt describing what is and is not suspicious activity to provide context for GenAI summaries."
|
||||
|
||||
Reference in New Issue
Block a user