mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-10-28 17:34:41 +03:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3725ad142f | ||
|
|
ee80129086 |
@@ -0,0 +1,9 @@
|
|||||||
|
{
|
||||||
|
"features": {
|
||||||
|
"ghcr.io/devcontainers/features/common-utils:2": {
|
||||||
|
"version": "2.7.1",
|
||||||
|
"resolved": "ghcr.io/devcontainers/features/common-utils@sha256:e76a6a8a35d1ad824daad1540725e0ec9f8ed2dc00d2850e990a9830013c7d08",
|
||||||
|
"integrity": "sha256:e76a6a8a35d1ad824daad1540725e0ec9f8ed2dc00d2850e990a9830013c7d08"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -504,6 +504,9 @@ review:
|
|||||||
# Presets adjust the tone and level of detail of the user-facing title,
|
# Presets adjust the tone and level of detail of the user-facing title,
|
||||||
# summary, and scene description; "default" leaves the built-in prompt unchanged.
|
# summary, and scene description; "default" leaves the built-in prompt unchanged.
|
||||||
response_style: default
|
response_style: default
|
||||||
|
# Optional: Let the model think before writing the description (default: shown below)
|
||||||
|
# Only applies to models with a per-request thinking toggle (ollama and llamacpp providers).
|
||||||
|
enable_thinking: False
|
||||||
# Optional: Save thumbnails sent to the GenAI provider for review/debugging purposes (default: shown below)
|
# Optional: Save thumbnails sent to the GenAI provider for review/debugging purposes (default: shown below)
|
||||||
debug_save_thumbnails: False
|
debug_save_thumbnails: False
|
||||||
|
|
||||||
|
|||||||
@@ -75,12 +75,11 @@ Running Generative AI models on CPU is not recommended, as high inference times
|
|||||||
|
|
||||||
You must use a vision-capable model with Frigate. The following models are recommended for local deployment of the `descriptions` and `chat` roles:
|
You must use a vision-capable model with Frigate. The following models are recommended for local deployment of the `descriptions` and `chat` roles:
|
||||||
|
|
||||||
| Model | Review [frame mode](/configuration/genai/genai_review#frame-mode) | Review [response style](/configuration/genai/genai_review#response-style) | Notes |
|
| Model | Review [frame mode](/configuration/genai/genai_review#frame-mode) | Notes |
|
||||||
| ------------------- | ----------------------------------------------------------------- | ------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
| ------------------- | --------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
||||||
| `qwen3-vl` | `frames` | `natural` | Strong visual and situational understanding, enhanced ability to identify smaller objects and interactions with object. Follows a sequence of frames on its own. |
|
| `qwen3-vl` | `frames` | Strong visual and situational understanding, enhanced ability to identify smaller objects and interactions with object. Follows a sequence of frames on its own. |
|
||||||
| `qwen3.6`/`qwen3.8` | `frames` | `natural` | Strong situational understanding, but missing DeepStack from qwen3-vl leading to worse performance for identifying objects in people's hand and other small details. |
|
| `qwen3.6`/`qwen3.8` | `frames` | Strong situational understanding, but missing DeepStack from qwen3-vl leading to worse performance for identifying objects in people's hand and other small details. |
|
||||||
| `gemma4` | `annotated_frames` | `default` | Strong situational understanding, sometimes resorts to more vague terms like 'interacts' instead of assigning a specific action. Loses track of activity that repeats or reverses, so it benefits from annotated frames. |
|
| `gemma4` | `annotated_frames` | Strong situational understanding, sometimes resorts to more vague terms like 'interacts' instead of assigning a specific action. Loses track of activity that repeats or reverses, so it benefits from annotated frames. |
|
||||||
| `lfm3-vl` | `annotated_frames` | `concise` | Very efficient with fast inference, making it a good choice for less powerful hardware or high volumes of review items. |
|
|
||||||
|
|
||||||
#### Embedding models
|
#### Embedding models
|
||||||
|
|
||||||
@@ -96,7 +95,7 @@ The `embeddings` role needs a different kind of model. Text queries are matched
|
|||||||
The `transcribe` role needs a model that accepts audio input. A text-only or vision-only model cannot serve this role. The following are recommended for local deployment of the `transcribe` role:
|
The `transcribe` role needs a model that accepts audio input. A text-only or vision-only model cannot serve this role. The following are recommended for local deployment of the `transcribe` role:
|
||||||
|
|
||||||
| Model | Notes |
|
| Model | Notes |
|
||||||
| ----------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
| ----------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
|
||||||
| `qwen3-asr` | Dedicated speech recognition model covering 30 languages, and the better choice for transcription quality. It only transcribes, so it cannot be shared with the `descriptions` or `chat` roles. |
|
| `qwen3-asr` | Dedicated speech recognition model covering 30 languages, and the better choice for transcription quality. It only transcribes, so it cannot be shared with the `descriptions` or `chat` roles. |
|
||||||
| `gemma4` | General multimodal model that accepts audio as well as images, so one served model can cover `transcribe` alongside the other roles. Transcript quality is below `qwen3-asr`, particularly on noisy audio. |
|
| `gemma4` | General multimodal model that accepts audio as well as images, so one served model can cover `transcribe` alongside the other roles. Transcript quality is below `qwen3-asr`, particularly on noisy audio. |
|
||||||
|
|
||||||
@@ -120,7 +119,7 @@ Vision-language models come in **instruct** variants (fine-tuned to follow instr
|
|||||||
|
|
||||||
Frigate manages reasoning per task automatically:
|
Frigate manages reasoning per task automatically:
|
||||||
|
|
||||||
- **Description tasks** (object descriptions, review descriptions, review summaries) are synthesis-only and benefit from concise, direct output, so Frigate disables thinking for these calls when the model exposes a per-request toggle.
|
- **Description tasks** (object descriptions, review descriptions, review summaries) are synthesis-only and benefit from concise, direct output, so Frigate disables thinking for these calls when the model exposes a per-request toggle. Review descriptions can opt back in with [`enable_thinking`](/configuration/genai/genai_review#thinking) for models trained to reason over video.
|
||||||
- **Chat** lets you toggle thinking on or off from the composer when the configured model supports it.
|
- **Chat** lets you toggle thinking on or off from the composer when the configured model supports it.
|
||||||
|
|
||||||
You can use a pure instruct, hybrid, or thinking-capable model with Frigate. No extra configuration is required to disable thinking for descriptions.
|
You can use a pure instruct, hybrid, or thinking-capable model with Frigate. No extra configuration is required to disable thinking for descriptions.
|
||||||
@@ -511,7 +510,7 @@ Frigate's Generative AI features are configured and enabled separately. [Review
|
|||||||
|
|
||||||
5. Save the exact images and prompts that were sent to your provider.
|
5. Save the exact images and prompts that were sent to your provider.
|
||||||
- Turn on **Save thumbnails** for the feature you are debugging (`review.genai.debug_save_thumbnails` or `objects.genai.debug_save_thumbnails`). Both features write to `/media/frigate/clips/genai-requests/`, and these files are admin-only.
|
- Turn on **Save thumbnails** for the feature you are debugging (`review.genai.debug_save_thumbnails` or `objects.genai.debug_save_thumbnails`). Both features write to `/media/frigate/clips/genai-requests/`, and these files are admin-only.
|
||||||
- Review descriptions write `genai-requests/<review_id>/` containing the numbered frames that were sent, plus `prompt.txt` and `response.txt` with the exact prompt and the raw, unparsed model response.
|
- Review descriptions write `genai-requests/<review_id>/` containing the numbered frames that were sent, plus `prompt.txt` and `response.txt` with the exact prompt and the raw, unparsed model response. When the provider returns a reasoning trace (for example with [thinking](/configuration/genai/genai_review#thinking) enabled), it is saved to `reasoning.txt`.
|
||||||
- Review summary reports write `genai-requests/<start_ts>-<end_ts>/prompt.txt` and `response.txt`. No images are involved, since a report summarizes existing review descriptions.
|
- Review summary reports write `genai-requests/<start_ts>-<end_ts>/prompt.txt` and `response.txt`. No images are involved, since a report summarizes existing review descriptions.
|
||||||
- Object descriptions write `genai-requests/<event_id>/` containing the numbered thumbnails. The prompt for object descriptions is not written to a file, it is only visible in the debug logs from step 4.
|
- Object descriptions write `genai-requests/<event_id>/` containing the numbered thumbnails. The prompt for object descriptions is not written to a file, it is only visible in the debug logs from step 4.
|
||||||
- Look at the saved images before blaming the model. If the object is small, blurry, or out of frame, no prompt will fix the result. For object descriptions, consider turning on **Use snapshots** (`objects.genai.use_snapshot`) to send a higher quality image. For review items, consider setting **Review image source** (`review.genai.image_source`) to `recordings` for 480p frames instead of the lower resolution preview frames.
|
- Look at the saved images before blaming the model. If the object is small, blurry, or out of frame, no prompt will fix the result. For object descriptions, consider turning on **Use snapshots** (`objects.genai.use_snapshot`) to send a higher quality image. For review items, consider setting **Review image source** (`review.genai.image_source`) to `recordings` for 480p frames instead of the lower resolution preview frames.
|
||||||
|
|||||||
@@ -264,6 +264,36 @@ review:
|
|||||||
</TabItem>
|
</TabItem>
|
||||||
</ConfigTabs>
|
</ConfigTabs>
|
||||||
|
|
||||||
|
### Thinking
|
||||||
|
|
||||||
|
By default, Frigate turns thinking off for review descriptions, since most models write better descriptions without spending tokens on a reasoning trace. Some models are trained to reason over video, and may follow complex scenes more accurately when allowed to think first. The `enable_thinking` option lets you turn thinking on for review descriptions to evaluate this with your model.
|
||||||
|
|
||||||
|
`enable_thinking` only applies to models that expose a per-request thinking toggle with the `ollama` or `llamacpp` providers. For llama.cpp, the model's chat template must use the `enable_thinking` variable. Other models keep their default behavior. With `openai`, use `runtime_options` (for example `reasoning_effort`) to control reasoning instead.
|
||||||
|
|
||||||
|
Thinking makes each review description take longer and use more tokens. If responses come back empty, the model may be using its whole output budget on reasoning.
|
||||||
|
|
||||||
|
When **Save thumbnails** (`debug_save_thumbnails`) is enabled, the reasoning trace is saved to `reasoning.txt` alongside `prompt.txt` and `response.txt`, so you can see what the model considered before answering. The trace is saved even when the model returns no answer. To compare a saved example with thinking on and off, use `testing-scripts/genai_review_tester.py` in the Frigate repository.
|
||||||
|
|
||||||
|
<ConfigTabs>
|
||||||
|
<TabItem value="ui">
|
||||||
|
|
||||||
|
Navigate to <NavPath path="Settings > Global configuration > Review" />.
|
||||||
|
|
||||||
|
- Enable **GenAI config > Enable thinking**. This option is only shown when the provider assigned the descriptions role is `ollama` or `llamacpp`.
|
||||||
|
|
||||||
|
</TabItem>
|
||||||
|
<TabItem value="yaml">
|
||||||
|
|
||||||
|
```yaml {4}
|
||||||
|
review:
|
||||||
|
genai:
|
||||||
|
enabled: true
|
||||||
|
enable_thinking: true
|
||||||
|
```
|
||||||
|
|
||||||
|
</TabItem>
|
||||||
|
</ConfigTabs>
|
||||||
|
|
||||||
## Review Reports
|
## Review Reports
|
||||||
|
|
||||||
Along with individual review item summaries, Generative AI can also produce a single report of review items from all cameras marked "suspicious" over a specified time period (for example, a daily summary of suspicious activity while you're on vacation).
|
Along with individual review item summaries, Generative AI can also produce a single report of review items from all cameras marked "suspicious" over a specified time period (for example, a daily summary of suspicious activity while you're on vacation).
|
||||||
|
|||||||
@@ -171,6 +171,11 @@ class GenAIReviewConfig(FrigateBaseModel):
|
|||||||
title="Response style",
|
title="Response style",
|
||||||
description="Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged.",
|
description="Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged.",
|
||||||
)
|
)
|
||||||
|
enable_thinking: bool = Field(
|
||||||
|
default=False,
|
||||||
|
title="Enable thinking",
|
||||||
|
description="Let the model reason before writing the review description. Only applies to models with a per-request thinking toggle (Ollama and llama.cpp providers); other models keep their default behavior. Thinking increases response time and token usage. When thumbnail saving is enabled, the reasoning is saved alongside the response.",
|
||||||
|
)
|
||||||
activity_context_prompt: str = Field(
|
activity_context_prompt: str = Field(
|
||||||
default="""### Normal Activity Indicators (Level 0)
|
default="""### Normal Activity Indicators (Level 0)
|
||||||
- Known/verified people in any zone at any time
|
- Known/verified people in any zone at any time
|
||||||
|
|||||||
@@ -12,6 +12,8 @@ import math
|
|||||||
from collections.abc import Sequence
|
from collections.abc import Sequence
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
|
from titlecase import titlecase
|
||||||
|
|
||||||
from frigate.models import Event, Timeline
|
from frigate.models import Event, Timeline
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -78,7 +80,13 @@ def describe_heading(dx: float, dy: float) -> str:
|
|||||||
|
|
||||||
|
|
||||||
def event_name(event: dict[str, Any]) -> str:
|
def event_name(event: dict[str, Any]) -> str:
|
||||||
"""Name an object for the notes, e.g. 'a person' or 'waste bin "Compost"'.
|
"""Name an object for the notes, e.g. 'a person' or 'Nick'.
|
||||||
|
|
||||||
|
A recognized sub_label is used on its own, in the same title case as
|
||||||
|
"Objects in Scene", so the notes name the subject the way the response
|
||||||
|
should. The object type is left out: leading with it makes models copy
|
||||||
|
"person" and drop the name from titles. An unrecognized face is stored as
|
||||||
|
"unknown" and stays unnamed.
|
||||||
|
|
||||||
Objects are never numbered or given track identifiers. Frigate opens a new
|
Objects are never numbered or given track identifiers. Frigate opens a new
|
||||||
tracked object whenever a subject is re-detected, so the tracking data
|
tracked object whenever a subject is re-detected, so the tracking data
|
||||||
@@ -86,10 +94,10 @@ def event_name(event: dict[str, Any]) -> str:
|
|||||||
ambiguous rather than implying either answer.
|
ambiguous rather than implying either answer.
|
||||||
"""
|
"""
|
||||||
label = str(event["label"]).replace("_", " ").replace("-verified", "")
|
label = str(event["label"]).replace("_", " ").replace("-verified", "")
|
||||||
sub_label = event.get("sub_label")
|
sub_label = str(event.get("sub_label") or "").strip()
|
||||||
|
|
||||||
if sub_label:
|
if sub_label and sub_label.lower() != "unknown":
|
||||||
return f'{label} "{sub_label}"'
|
return titlecase(sub_label.replace("_", " "))
|
||||||
|
|
||||||
article = "an" if label[:1].lower() in "aeiou" else "a"
|
article = "an" if label[:1].lower() in "aeiou" else "a"
|
||||||
return f"{article} {label}"
|
return f"{article} {label}"
|
||||||
@@ -387,11 +395,10 @@ def build_frame_captions(
|
|||||||
|
|
||||||
# audio and manual review items can have state changes but no tracked objects
|
# audio and manual review items can have state changes but no tracked objects
|
||||||
if events:
|
if events:
|
||||||
|
# No source prefix. A tag placed before the name is read as the
|
||||||
|
# subject's identity.
|
||||||
timeline.extend(
|
timeline.extend(
|
||||||
(timestamp, f"[tracker] {note}")
|
build_timeline(events, span_end, get_state_changes(detection_ids))
|
||||||
for timestamp, note in build_timeline(
|
|
||||||
events, span_end, get_state_changes(detection_ids)
|
|
||||||
)
|
|
||||||
)
|
)
|
||||||
|
|
||||||
timeline.extend(
|
timeline.extend(
|
||||||
|
|||||||
@@ -813,6 +813,7 @@ def run_analysis(
|
|||||||
genai_config.activity_context_prompt,
|
genai_config.activity_context_prompt,
|
||||||
genai_config.response_style,
|
genai_config.response_style,
|
||||||
frame_captions,
|
frame_captions,
|
||||||
|
genai_config.enable_thinking,
|
||||||
)
|
)
|
||||||
review_inference_speed.update(datetime.datetime.now().timestamp() - start)
|
review_inference_speed.update(datetime.datetime.now().timestamp() - start)
|
||||||
|
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import os
|
|||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
from collections.abc import AsyncGenerator, Callable
|
from collections.abc import AsyncGenerator, Callable
|
||||||
|
from dataclasses import dataclass
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
import numpy as np
|
import numpy as np
|
||||||
@@ -33,6 +34,7 @@ __all__ = [
|
|||||||
"GenAIClientManager",
|
"GenAIClientManager",
|
||||||
"GenAIConfig",
|
"GenAIConfig",
|
||||||
"GenAIProviderEnum",
|
"GenAIProviderEnum",
|
||||||
|
"GenAIResponse",
|
||||||
"PROVIDERS",
|
"PROVIDERS",
|
||||||
"load_providers",
|
"load_providers",
|
||||||
"register_genai_provider",
|
"register_genai_provider",
|
||||||
@@ -41,6 +43,18 @@ __all__ = [
|
|||||||
PROVIDERS: dict[GenAIProviderEnum, type["GenAIClient"]] = {}
|
PROVIDERS: dict[GenAIProviderEnum, type["GenAIClient"]] = {}
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass
|
||||||
|
class GenAIResponse:
|
||||||
|
"""A provider's answer to a single request.
|
||||||
|
|
||||||
|
The reasoning trace is kept apart from the content so callers can save it
|
||||||
|
for debugging without it ending up in the parsed answer.
|
||||||
|
"""
|
||||||
|
|
||||||
|
content: str
|
||||||
|
reasoning: str | None = None
|
||||||
|
|
||||||
|
|
||||||
def register_genai_provider(key: GenAIProviderEnum) -> Callable:
|
def register_genai_provider(key: GenAIProviderEnum) -> Callable:
|
||||||
"""Register a GenAI provider."""
|
"""Register a GenAI provider."""
|
||||||
|
|
||||||
@@ -109,11 +123,14 @@ class GenAIClient:
|
|||||||
activity_context_prompt: str,
|
activity_context_prompt: str,
|
||||||
response_style: str = "default",
|
response_style: str = "default",
|
||||||
frame_captions: list[str] | None = None,
|
frame_captions: list[str] | None = None,
|
||||||
|
enable_thinking: bool = False,
|
||||||
) -> ReviewMetadata | None:
|
) -> ReviewMetadata | None:
|
||||||
"""Generate a description for the review item activity.
|
"""Generate a description for the review item activity.
|
||||||
|
|
||||||
`frame_captions` holds one caption per thumbnail for the annotated
|
`frame_captions` holds one caption per thumbnail for the annotated
|
||||||
frame mode; each is sent directly before its frame.
|
frame mode; each is sent directly before its frame. `enable_thinking`
|
||||||
|
lets the model reason before answering, when the provider exposes a
|
||||||
|
per-request toggle.
|
||||||
"""
|
"""
|
||||||
if frame_captions and len(frame_captions) != len(thumbnails):
|
if frame_captions and len(frame_captions) != len(thumbnails):
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -137,6 +154,13 @@ class GenAIClient:
|
|||||||
f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}"
|
f"Sending {len(thumbnails)} images to create review description on {review_data['camera']}"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
if enable_thinking and not self.supports_toggleable_thinking:
|
||||||
|
logger.debug(
|
||||||
|
"Thinking requested for review description but model %s does not "
|
||||||
|
"expose a per-request thinking toggle, using its default behavior",
|
||||||
|
self.genai_config.model,
|
||||||
|
)
|
||||||
|
|
||||||
if debug_save:
|
if debug_save:
|
||||||
with open(
|
with open(
|
||||||
os.path.join(
|
os.path.join(
|
||||||
@@ -164,12 +188,14 @@ class GenAIClient:
|
|||||||
|
|
||||||
response_format = build_review_description_response_format(concerns)
|
response_format = build_review_description_response_format(concerns)
|
||||||
|
|
||||||
response = self._send(
|
result = self._send(
|
||||||
context_prompt,
|
context_prompt,
|
||||||
thumbnails,
|
thumbnails,
|
||||||
response_format,
|
response_format,
|
||||||
|
enable_thinking=enable_thinking,
|
||||||
image_captions=frame_captions,
|
image_captions=frame_captions,
|
||||||
)
|
)
|
||||||
|
response = result.content if result else None
|
||||||
|
|
||||||
if debug_save and response:
|
if debug_save and response:
|
||||||
with open(
|
with open(
|
||||||
@@ -180,6 +206,17 @@ class GenAIClient:
|
|||||||
) as f:
|
) as f:
|
||||||
f.write(response)
|
f.write(response)
|
||||||
|
|
||||||
|
# Saved even when the content is blank, since a model that spends its
|
||||||
|
# whole output budget thinking returns a trace but no answer.
|
||||||
|
if debug_save and result and result.reasoning:
|
||||||
|
with open(
|
||||||
|
os.path.join(
|
||||||
|
CLIPS_DIR, "genai-requests", review_data["id"], "reasoning.txt"
|
||||||
|
),
|
||||||
|
"w",
|
||||||
|
) as f:
|
||||||
|
f.write(result.reasoning)
|
||||||
|
|
||||||
if response:
|
if response:
|
||||||
clean_json = re.sub(
|
clean_json = re.sub(
|
||||||
r"\n?```$", "", re.sub(r"^```[a-zA-Z0-9]*\n?", "", response)
|
r"\n?```$", "", re.sub(r"^```[a-zA-Z0-9]*\n?", "", response)
|
||||||
@@ -268,7 +305,8 @@ class GenAIClient:
|
|||||||
) as f:
|
) as f:
|
||||||
f.write(timeline_summary_prompt)
|
f.write(timeline_summary_prompt)
|
||||||
|
|
||||||
response = self._send(timeline_summary_prompt, [])
|
result = self._send(timeline_summary_prompt, [])
|
||||||
|
response = result.content if result else None
|
||||||
|
|
||||||
if debug_save and response:
|
if debug_save and response:
|
||||||
with open(
|
with open(
|
||||||
@@ -295,7 +333,8 @@ class GenAIClient:
|
|||||||
return None
|
return None
|
||||||
|
|
||||||
logger.debug(f"Sending images to genai provider with prompt: {prompt}")
|
logger.debug(f"Sending images to genai provider with prompt: {prompt}")
|
||||||
return self._send(prompt, thumbnails)
|
result = self._send(prompt, thumbnails)
|
||||||
|
return result.content if result else None
|
||||||
|
|
||||||
def _init_provider(self) -> Any:
|
def _init_provider(self) -> Any:
|
||||||
"""Initialize the client."""
|
"""Initialize the client."""
|
||||||
@@ -308,13 +347,16 @@ class GenAIClient:
|
|||||||
response_format: dict | None = None,
|
response_format: dict | None = None,
|
||||||
enable_thinking: bool = False,
|
enable_thinking: bool = False,
|
||||||
image_captions: list[str] | None = None,
|
image_captions: list[str] | None = None,
|
||||||
) -> str | None:
|
) -> GenAIResponse | None:
|
||||||
"""Submit a request to the provider.
|
"""Submit a request to the provider.
|
||||||
|
|
||||||
``enable_thinking`` is honored only by providers that report
|
``enable_thinking`` is honored only by providers that report
|
||||||
``supports_toggleable_thinking``. Description-style callers leave it
|
``supports_toggleable_thinking``. Description-style callers leave it
|
||||||
at the default (off) since synthesis tasks don't benefit from
|
at the default (off) unless the user opts in, since most models write
|
||||||
reasoning traces.
|
better descriptions without spending tokens on a reasoning trace.
|
||||||
|
|
||||||
|
Returns None when the request fails. Providers that surface a
|
||||||
|
reasoning trace separately return it in ``reasoning``.
|
||||||
|
|
||||||
``image_captions`` carries one caption per image, to be placed
|
``image_captions`` carries one caption per image, to be placed
|
||||||
immediately before its image so the model can tell the frames apart.
|
immediately before its image so the model can tell the frames apart.
|
||||||
|
|||||||
@@ -12,7 +12,7 @@ from google.genai import errors, types
|
|||||||
from google.genai.types import FunctionCallingConfigMode
|
from google.genai.types import FunctionCallingConfigMode
|
||||||
|
|
||||||
from frigate.config import GenAIProviderEnum
|
from frigate.config import GenAIProviderEnum
|
||||||
from frigate.genai import GenAIClient, register_genai_provider
|
from frigate.genai import GenAIClient, GenAIResponse, register_genai_provider
|
||||||
from frigate.genai.utils import interleave_images
|
from frigate.genai.utils import interleave_images
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -124,8 +124,13 @@ class GeminiClient(GenAIClient):
|
|||||||
response_format: dict | None = None,
|
response_format: dict | None = None,
|
||||||
enable_thinking: bool = False,
|
enable_thinking: bool = False,
|
||||||
image_captions: list[str] | None = None,
|
image_captions: list[str] | None = None,
|
||||||
) -> str | None:
|
) -> GenAIResponse | None:
|
||||||
"""Submit a request to Gemini."""
|
"""Submit a request to Gemini.
|
||||||
|
|
||||||
|
Thinking is configured at the model level for Gemini (or through
|
||||||
|
``thinking_config`` in ``runtime_options``), so ``enable_thinking`` is
|
||||||
|
ignored here.
|
||||||
|
"""
|
||||||
contents: list[Any] = [
|
contents: list[Any] = [
|
||||||
part
|
part
|
||||||
if isinstance(part, str)
|
if isinstance(part, str)
|
||||||
@@ -165,7 +170,7 @@ class GeminiClient(GenAIClient):
|
|||||||
except (ValueError, AttributeError):
|
except (ValueError, AttributeError):
|
||||||
# No description was generated
|
# No description was generated
|
||||||
return None
|
return None
|
||||||
return description
|
return GenAIResponse(description)
|
||||||
|
|
||||||
@property
|
@property
|
||||||
def supports_transcription(self) -> bool:
|
def supports_transcription(self) -> bool:
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ import numpy as np
|
|||||||
import requests
|
import requests
|
||||||
|
|
||||||
from frigate.config import GenAIProviderEnum
|
from frigate.config import GenAIProviderEnum
|
||||||
from frigate.genai import GenAIClient, register_genai_provider
|
from frigate.genai import GenAIClient, GenAIResponse, register_genai_provider
|
||||||
from frigate.genai.utils import (
|
from frigate.genai.utils import (
|
||||||
interleave_images,
|
interleave_images,
|
||||||
parse_tool_calls_from_message,
|
parse_tool_calls_from_message,
|
||||||
@@ -304,7 +304,7 @@ class LlamaCppClient(GenAIClient):
|
|||||||
response_format: dict | None = None,
|
response_format: dict | None = None,
|
||||||
enable_thinking: bool = False,
|
enable_thinking: bool = False,
|
||||||
image_captions: list[str] | None = None,
|
image_captions: list[str] | None = None,
|
||||||
) -> str | None:
|
) -> GenAIResponse | None:
|
||||||
"""Submit a request to llama.cpp server."""
|
"""Submit a request to llama.cpp server."""
|
||||||
if self.provider is None:
|
if self.provider is None:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -362,7 +362,8 @@ class LlamaCppClient(GenAIClient):
|
|||||||
):
|
):
|
||||||
choice = result["choices"][0]
|
choice = result["choices"][0]
|
||||||
if "message" in choice and "content" in choice["message"]:
|
if "message" in choice and "content" in choice["message"]:
|
||||||
return str(choice["message"]["content"].strip())
|
message = self._message_from_choice(choice)
|
||||||
|
return GenAIResponse(message["content"] or "", message["reasoning"])
|
||||||
return None
|
return None
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logger.warning("llama.cpp returned an error: %s", str(e))
|
logger.warning("llama.cpp returned an error: %s", str(e))
|
||||||
|
|||||||
@@ -14,7 +14,7 @@ from ollama import Client as ApiClient
|
|||||||
from ollama import ResponseError
|
from ollama import ResponseError
|
||||||
|
|
||||||
from frigate.config import GenAIProviderEnum
|
from frigate.config import GenAIProviderEnum
|
||||||
from frigate.genai import GenAIClient, register_genai_provider
|
from frigate.genai import GenAIClient, GenAIResponse, register_genai_provider
|
||||||
from frigate.genai.utils import (
|
from frigate.genai.utils import (
|
||||||
interleave_images,
|
interleave_images,
|
||||||
parse_tool_calls_from_message,
|
parse_tool_calls_from_message,
|
||||||
@@ -243,7 +243,7 @@ class OllamaClient(GenAIClient):
|
|||||||
response_format: dict | None = None,
|
response_format: dict | None = None,
|
||||||
enable_thinking: bool = False,
|
enable_thinking: bool = False,
|
||||||
image_captions: list[str] | None = None,
|
image_captions: list[str] | None = None,
|
||||||
) -> str | None:
|
) -> GenAIResponse | None:
|
||||||
"""Submit a request to Ollama through the chat API, the same path the
|
"""Submit a request to Ollama through the chat API, the same path the
|
||||||
tool-calling chat uses, with image placeholders keeping any captions
|
tool-calling chat uses, with image placeholders keeping any captions
|
||||||
next to their frames."""
|
next to their frames."""
|
||||||
@@ -299,7 +299,8 @@ class OllamaClient(GenAIClient):
|
|||||||
response.get("eval_count"),
|
response.get("eval_count"),
|
||||||
response.get("prompt_eval_count"),
|
response.get("prompt_eval_count"),
|
||||||
)
|
)
|
||||||
response_text = self._message_from_response(response)["content"] or ""
|
message_result = self._message_from_response(response)
|
||||||
|
response_text = message_result["content"] or ""
|
||||||
|
|
||||||
if not response_text:
|
if not response_text:
|
||||||
logger.warning(
|
logger.warning(
|
||||||
@@ -310,7 +311,7 @@ class OllamaClient(GenAIClient):
|
|||||||
response.get("eval_count"),
|
response.get("eval_count"),
|
||||||
)
|
)
|
||||||
|
|
||||||
return response_text
|
return GenAIResponse(response_text, message_result["reasoning"])
|
||||||
|
|
||||||
def list_models(self) -> list[str]:
|
def list_models(self) -> list[str]:
|
||||||
"""Return available model names from the Ollama server."""
|
"""Return available model names from the Ollama server."""
|
||||||
|
|||||||
@@ -10,7 +10,7 @@ from httpx import TimeoutException
|
|||||||
from openai import OpenAI
|
from openai import OpenAI
|
||||||
|
|
||||||
from frigate.config import GenAIProviderEnum
|
from frigate.config import GenAIProviderEnum
|
||||||
from frigate.genai import GenAIClient, register_genai_provider
|
from frigate.genai import GenAIClient, GenAIResponse, register_genai_provider
|
||||||
from frigate.genai.utils import interleave_images
|
from frigate.genai.utils import interleave_images
|
||||||
|
|
||||||
logger = logging.getLogger(__name__)
|
logger = logging.getLogger(__name__)
|
||||||
@@ -71,8 +71,14 @@ class OpenAIClient(GenAIClient):
|
|||||||
response_format: dict | None = None,
|
response_format: dict | None = None,
|
||||||
enable_thinking: bool = False,
|
enable_thinking: bool = False,
|
||||||
image_captions: list[str] | None = None,
|
image_captions: list[str] | None = None,
|
||||||
) -> str | None:
|
) -> GenAIResponse | None:
|
||||||
"""Submit a request to OpenAI."""
|
"""Submit a request to OpenAI.
|
||||||
|
|
||||||
|
The chat completions API has no per-request thinking toggle, so
|
||||||
|
``enable_thinking`` is ignored here. Reasoning is controlled through
|
||||||
|
``runtime_options`` (e.g. ``reasoning_effort``), and any trace the
|
||||||
|
server returns separately is passed back as ``reasoning``.
|
||||||
|
"""
|
||||||
messages_content: list[dict] = []
|
messages_content: list[dict] = []
|
||||||
for part in interleave_images(prompt, images, image_captions):
|
for part in interleave_images(prompt, images, image_captions):
|
||||||
if isinstance(part, str):
|
if isinstance(part, str):
|
||||||
@@ -120,6 +126,10 @@ class OpenAIClient(GenAIClient):
|
|||||||
):
|
):
|
||||||
message = result.choices[0].message
|
message = result.choices[0].message
|
||||||
content = message.content
|
content = message.content
|
||||||
|
raw_reasoning = getattr(message, "reasoning_content", None) or getattr(
|
||||||
|
message, "reasoning", None
|
||||||
|
)
|
||||||
|
reasoning = raw_reasoning.strip() if raw_reasoning else None
|
||||||
|
|
||||||
if not content:
|
if not content:
|
||||||
# When reasoning is enabled for some OpenAI backends the actual response
|
# When reasoning is enabled for some OpenAI backends the actual response
|
||||||
@@ -134,7 +144,9 @@ class OpenAIClient(GenAIClient):
|
|||||||
)
|
)
|
||||||
content = reasoning_content
|
content = reasoning_content
|
||||||
|
|
||||||
return str(content.strip()) if content else None
|
return (
|
||||||
|
GenAIResponse(str(content.strip()), reasoning) if content else None
|
||||||
|
)
|
||||||
return None
|
return None
|
||||||
except (TimeoutException, Exception) as e:
|
except (TimeoutException, Exception) as e:
|
||||||
logger.warning("OpenAI returned an error: %s", str(e))
|
logger.warning("OpenAI returned an error: %s", str(e))
|
||||||
|
|||||||
+15
-15
@@ -19,10 +19,10 @@ from frigate.models import Event
|
|||||||
# Base guidance per response field. `observations` is a reasoning scaffold,
|
# Base guidance per response field. `observations` is a reasoning scaffold,
|
||||||
# not user-facing, so style presets never override it.
|
# not user-facing, so style presets never override it.
|
||||||
REVIEW_DESCRIPTION_FIELD_GUIDELINES: dict[str, str] = {
|
REVIEW_DESCRIPTION_FIELD_GUIDELINES: dict[str, str] = {
|
||||||
"observations": "Include the very start of the activity — for example, a vehicle entering the frame or pulling into the driveway — even if it lasts only a few frames and the rest of the clip is dominated by a longer activity. Include each arrival, departure, object handled, and notable change in position or state. Each item is a single concrete fact written as a complete sentence.",
|
"observations": "Include the very start of the activity, for example a vehicle entering the frame or pulling into the driveway, even if it lasts only a few frames and the rest of the clip is dominated by a longer activity. Include each arrival, departure, object handled, and notable change in position or state. Each item is a single concrete fact written as a complete sentence. Use every name from Objects in Scene.",
|
||||||
"scene": 'Describe how the sequence begins, then the progression of events — all significant movements and actions in order. For example, if a vehicle arrives and then a person exits, describe both sequentially. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name — do not replace them with generic terms. For unnamed objects (e.g., "person", "car"), refer to them naturally with articles (e.g., "a person", "the car"). Your description should align with and support the threat level you assign.',
|
"scene": 'Describe how the sequence begins, then the progression of events, all significant movements and actions in order. For example, if a vehicle arrives and then a person exits, describe both sequentially. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name. For unnamed objects (e.g., "person", "car"), refer to them naturally with articles (e.g., "a person", "the car"). Your description should align with and support the threat level you assign.',
|
||||||
"title": "Name the primary activity across the observations, together with the location. An activity is what is being done with objects, tools, or surfaces; locomotion through the scene qualifies as the activity only when no other interaction is observed. For named subjects, always use their name. For unnamed objects, refer to them naturally with articles.",
|
"title": "Every name in Objects in Scene belongs in the title. Name the primary activity across the observations, together with the location. An activity is what is being done with objects, tools, or surfaces; locomotion through the scene qualifies as the activity only when no other interaction is observed. For an unnamed object, refer to it naturally with an article.",
|
||||||
"shortSummary": "Briefly summarize the primary activity across the observations.",
|
"shortSummary": "Briefly summarize the primary activity across the observations. Use every name from Objects in Scene.",
|
||||||
"potential_threat_level": "Must be consistent with your scene description and the activity patterns above.",
|
"potential_threat_level": "Must be consistent with your scene description and the activity patterns above.",
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -31,19 +31,19 @@ REVIEW_DESCRIPTION_FIELD_GUIDELINES: dict[str, str] = {
|
|||||||
# competing style instructions.
|
# competing style instructions.
|
||||||
REVIEW_RESPONSE_STYLES: dict[str, dict[str, str]] = {
|
REVIEW_RESPONSE_STYLES: dict[str, dict[str, str]] = {
|
||||||
"natural": {
|
"natural": {
|
||||||
"scene": 'Recount what happened the way a person would describe it to a neighbor, in plain everyday language: how the sequence begins, then each significant movement and action in the order it happens. Use flowing sentences that connect related actions, written the way people actually talk rather than like a surveillance report. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name rather than a generic term. Refer to unnamed objects naturally with articles. Stay factual, and keep the description consistent with the threat level you assign.',
|
"scene": 'Recount what happened the way a person would describe it to a neighbor, in plain everyday language: how the sequence begins, then each significant movement and action in the order it happens. Use flowing sentences that connect related actions, written the way people actually talk rather than like a surveillance report. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name. Refer to unnamed objects naturally with articles. Stay factual, and keep the description consistent with the threat level you assign.',
|
||||||
"title": 'Write the title as a short, sentence-case headline in present tense: the subject, then what they do, phrased the way you would text it to the homeowner. Describe only the action you see; do not assign the person a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. Name the main thing done, not just movement through the scene, unless movement is all that happens. For named subjects, always use their name.',
|
"title": 'Every name in Objects in Scene belongs in the title. Write the title as a short, sentence-case headline in present tense: the subject, then what they do, phrased the way you would text it to the homeowner. Describe only the action you see; do not assign the person a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. Name the main thing done, not just movement through the scene, unless movement is all that happens.',
|
||||||
"shortSummary": "Sum up the primary activity in one short, natural sentence, as if mentioning it to someone in passing.",
|
"shortSummary": "Sum up the primary activity in one short, natural sentence, as if mentioning it to someone in passing. Use every name from Objects in Scene.",
|
||||||
},
|
},
|
||||||
"concise": {
|
"concise": {
|
||||||
"scene": 'Cover each significant movement and action in order using as few short, direct sentences as possible, omitting environmental and cosmetic detail unless it affects the assessment. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign.',
|
"scene": 'Cover each significant movement and action in order using as few short, direct sentences as possible, omitting environmental and cosmetic detail unless it affects the assessment. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign.',
|
||||||
"title": 'Write the title as a terse label of two to four words naming the specific activity observed and where it happened. Do not assign a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. For named subjects, always use their name.',
|
"title": 'Every name in Objects in Scene belongs in the title. Write the title as a terse label of two to four words naming the specific activity observed and where it happened, apart from those names. Do not assign a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene.',
|
||||||
"shortSummary": "Summarize the primary activity in one short sentence.",
|
"shortSummary": "Summarize the primary activity in one short sentence. Use every name from Objects in Scene.",
|
||||||
},
|
},
|
||||||
"detailed": {
|
"detailed": {
|
||||||
"scene": 'Describe how the sequence begins, then the progression of events — all significant movements and actions in order, including the specifics that best identify the subjects: colors, clothing, carried items, positions, and paths of movement, plus environmental details like lighting changes when they stand out. Favor the most identifying details over exhaustive coverage, and keep every added detail observational rather than speculative. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name rather than a generic term. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign.',
|
"scene": 'Describe how the sequence begins, then the progression of events, all significant movements and actions in order, including the specifics that best identify the subjects: colors, clothing, carried items, positions, and paths of movement, plus environmental details like lighting changes when they stand out. Favor the most identifying details over exhaustive coverage, and keep every added detail observational rather than speculative. For named subjects (those with a `←` separator in "Objects in Scene"), always use their name. For unnamed objects, refer to them naturally with articles. Your description should align with and support the threat level you assign.',
|
||||||
"title": 'Write the title as a specific description of who did what and where, in under roughly twelve words, including the most distinguishing visible detail of the subject, such as clothing or vehicle color. Do not assign a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. Name the main thing done, not just movement through the scene, unless movement is all that happens. For named subjects, always use their name.',
|
"title": 'Every name in Objects in Scene belongs in the title. Write the title as a specific description of who did what and where, in under roughly twelve words, including the most distinguishing visible detail of the subject, such as clothing or vehicle color. Do not assign a role or purpose that is not visibly indicated by a uniform, a marked vehicle, or a "(delivery/service)" tag in Objects in Scene. Name the main thing done, not just movement through the scene, unless movement is all that happens.',
|
||||||
"shortSummary": "Briefly summarize the primary activity across the observations, including the most identifying visible detail, such as vehicle color or clothing.",
|
"shortSummary": "Briefly summarize the primary activity across the observations, including the most identifying visible detail, such as vehicle color or clothing. Use every name from Objects in Scene.",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -63,7 +63,7 @@ def get_review_field_guidelines(response_style: str = "default") -> dict[str, st
|
|||||||
# mode. Neither the notes nor this guidance say whether repeated detections are
|
# mode. Neither the notes nor this guidance say whether repeated detections are
|
||||||
# the same subject, since the tracking data cannot tell.
|
# the same subject, since the tracking data cannot tell.
|
||||||
FRAME_ANNOTATION_GUIDANCE = """- Each image below is immediately preceded by a text label giving its frame number and how many seconds into the sequence it was captured. Use these labels to track the order of events and the time between them.
|
FRAME_ANNOTATION_GUIDANCE = """- Each image below is immediately preceded by a text label giving its frame number and how many seconds into the sequence it was captured. Use these labels to track the order of events and the time between them.
|
||||||
- Some images below are preceded by notes from the camera's object tracker recording what changed at that point: an object being first detected, starting to move, reversing direction, stopping, or no longer being detected. These notes come from tracking data rather than from the images, and they are reliable. Use them to establish how many distinct activities occur and in what order, and describe every one of them."""
|
- Some images below are preceded by a note of what changed at that moment: a subject being first detected, starting to move, reversing direction, stopping, or no longer being detected. When the note names the subject, use that name. These notes come from tracking data rather than from the images, and they are reliable. Use them to establish how many distinct activities occur and in what order, and describe every one of them."""
|
||||||
|
|
||||||
|
|
||||||
def build_review_description_prompt(
|
def build_review_description_prompt(
|
||||||
@@ -164,9 +164,9 @@ Respond with a JSON object matching the provided schema. Field-specific guidance
|
|||||||
|
|
||||||
## Objects in Scene
|
## Objects in Scene
|
||||||
|
|
||||||
Each line represents a detection state, not necessarily unique individuals. The `←` symbol separates a recognized subject's name from their object type — use only the name (before the `←`) in your response, not the type after it. The same subject may appear across multiple lines if detected multiple times.
|
The `←` symbol separates a recognized, known subject's name from their object type. Use each of these names. The same subject may appear on more than one line. A line with no `←` has no name; refer to it naturally with an article.
|
||||||
|
|
||||||
**Note: Unidentified objects (without names) are NOT indicators of suspicious activity—they simply mean the system hasn't identified that object.**
|
An object with no name is not suspicious just because it is unnamed.
|
||||||
{get_objects_list()}
|
{get_objects_list()}
|
||||||
|
|
||||||
{get_language_prompt()}
|
{get_language_prompt()}
|
||||||
|
|||||||
@@ -83,6 +83,11 @@ class TestReviewResponseStyle(unittest.TestCase):
|
|||||||
prompt,
|
prompt,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def test_named_subject_is_the_title_subject(self):
|
||||||
|
prompt = self._build_prompt()
|
||||||
|
self.assertIn("Every name in Objects in Scene belongs in the title.", prompt)
|
||||||
|
self.assertIn("recognized, known subject", prompt)
|
||||||
|
|
||||||
|
|
||||||
class TestClassificationStateChanges(unittest.TestCase):
|
class TestClassificationStateChanges(unittest.TestCase):
|
||||||
def _build_prompt(self, **extra) -> str:
|
def _build_prompt(self, **extra) -> str:
|
||||||
|
|||||||
@@ -15,6 +15,8 @@ crashing message conversion.
|
|||||||
import asyncio
|
import asyncio
|
||||||
import base64
|
import base64
|
||||||
import json
|
import json
|
||||||
|
import os
|
||||||
|
import tempfile
|
||||||
import unittest
|
import unittest
|
||||||
from types import SimpleNamespace
|
from types import SimpleNamespace
|
||||||
from unittest.mock import AsyncMock, MagicMock, patch
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||||||
@@ -22,7 +24,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
|
|||||||
import requests
|
import requests
|
||||||
|
|
||||||
from frigate.config import GenAIConfig, GenAIProviderEnum
|
from frigate.config import GenAIConfig, GenAIProviderEnum
|
||||||
from frigate.genai import PROVIDERS, load_providers
|
from frigate.genai import PROVIDERS, GenAIClient, GenAIResponse, load_providers
|
||||||
|
|
||||||
load_providers()
|
load_providers()
|
||||||
|
|
||||||
@@ -194,6 +196,19 @@ class TestOpenAIProvider(unittest.TestCase):
|
|||||||
final = _final_message(_collect(client, MULTIMODAL_MESSAGES))
|
final = _final_message(_collect(client, MULTIMODAL_MESSAGES))
|
||||||
self.assertEqual(final["content"], "ok")
|
self.assertEqual(final["content"], "ok")
|
||||||
|
|
||||||
|
def test_send_returns_reasoning_content(self):
|
||||||
|
client = self._client()
|
||||||
|
message = SimpleNamespace(
|
||||||
|
content=" ok ", reasoning_content=" thought ", reasoning=None
|
||||||
|
)
|
||||||
|
client.provider.chat.completions.create = MagicMock(
|
||||||
|
return_value=SimpleNamespace(choices=[SimpleNamespace(message=message)])
|
||||||
|
)
|
||||||
|
|
||||||
|
result = client._send("prompt", [b"a"], enable_thinking=True)
|
||||||
|
|
||||||
|
self.assertEqual(result, GenAIResponse("ok", "thought"))
|
||||||
|
|
||||||
|
|
||||||
# ---------------------------------------------------------------------------
|
# ---------------------------------------------------------------------------
|
||||||
# Gemini
|
# Gemini
|
||||||
@@ -399,7 +414,7 @@ class TestOllamaProvider(unittest.TestCase):
|
|||||||
image_captions=["Frame 1 of 2", "Frame 2 of 2"],
|
image_captions=["Frame 1 of 2", "Frame 2 of 2"],
|
||||||
)
|
)
|
||||||
|
|
||||||
self.assertEqual(result, '{"ok": true}')
|
self.assertEqual(result, GenAIResponse('{"ok": true}'))
|
||||||
client.provider.generate.assert_not_called()
|
client.provider.generate.assert_not_called()
|
||||||
params = client.provider.chat.call_args.kwargs
|
params = client.provider.chat.call_args.kwargs
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
@@ -427,6 +442,30 @@ class TestOllamaProvider(unittest.TestCase):
|
|||||||
self.assertEqual(message["content"], "prompt\n[img]")
|
self.assertEqual(message["content"], "prompt\n[img]")
|
||||||
self.assertEqual(message["images"], [b"a"])
|
self.assertEqual(message["images"], [b"a"])
|
||||||
|
|
||||||
|
def test_send_with_thinking_returns_reasoning(self):
|
||||||
|
client = self._client()
|
||||||
|
client.provider = MagicMock()
|
||||||
|
client.provider.chat.return_value = {
|
||||||
|
"message": {"content": "ok", "thinking": " looked at each frame "},
|
||||||
|
"done": True,
|
||||||
|
}
|
||||||
|
client._supports_thinking_cache = True
|
||||||
|
|
||||||
|
result = client._send("prompt", [b"a"], enable_thinking=True)
|
||||||
|
|
||||||
|
self.assertTrue(client.provider.chat.call_args.kwargs["think"])
|
||||||
|
self.assertEqual(result, GenAIResponse("ok", "looked at each frame"))
|
||||||
|
|
||||||
|
def test_send_disables_thinking_by_default(self):
|
||||||
|
client = self._client()
|
||||||
|
client.provider = MagicMock()
|
||||||
|
client.provider.chat.return_value = {"message": {"content": "ok"}, "done": True}
|
||||||
|
client._supports_thinking_cache = True
|
||||||
|
|
||||||
|
client._send("prompt", [b"a"])
|
||||||
|
|
||||||
|
self.assertFalse(client.provider.chat.call_args.kwargs["think"])
|
||||||
|
|
||||||
def test_capabilities_drive_embeddings_and_thinking(self):
|
def test_capabilities_drive_embeddings_and_thinking(self):
|
||||||
client = self._client()
|
client = self._client()
|
||||||
client.provider = MagicMock()
|
client.provider = MagicMock()
|
||||||
@@ -696,6 +735,42 @@ class TestLlamaCppProvider(unittest.TestCase):
|
|||||||
final = _final_message(self._run_with_lines(client, lines, MULTIMODAL_MESSAGES))
|
final = _final_message(self._run_with_lines(client, lines, MULTIMODAL_MESSAGES))
|
||||||
self.assertEqual(final["content"], "ok")
|
self.assertEqual(final["content"], "ok")
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def _chat_response(message):
|
||||||
|
response = MagicMock()
|
||||||
|
response.json.return_value = {"choices": [{"message": message}]}
|
||||||
|
return response
|
||||||
|
|
||||||
|
def test_send_with_thinking_returns_reasoning(self):
|
||||||
|
client = self._client()
|
||||||
|
client._supports_reasoning = True
|
||||||
|
response = self._chat_response(
|
||||||
|
{"content": " ok ", "reasoning_content": " looked at each frame "}
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch.object(client, "_post", return_value=response) as post:
|
||||||
|
result = client._send("prompt", [b"a"], enable_thinking=True)
|
||||||
|
|
||||||
|
self.assertEqual(
|
||||||
|
post.call_args.kwargs["json"]["chat_template_kwargs"],
|
||||||
|
{"enable_thinking": True},
|
||||||
|
)
|
||||||
|
self.assertEqual(result, GenAIResponse("ok", "looked at each frame"))
|
||||||
|
|
||||||
|
def test_send_keeps_reasoning_when_content_is_null(self):
|
||||||
|
# A model that exhausts its output budget while thinking returns a
|
||||||
|
# trace with no content; the trace is what explains the blank answer.
|
||||||
|
client = self._client()
|
||||||
|
client._supports_reasoning = True
|
||||||
|
response = self._chat_response(
|
||||||
|
{"content": None, "reasoning_content": "still thinking"}
|
||||||
|
)
|
||||||
|
|
||||||
|
with patch.object(client, "_post", return_value=response):
|
||||||
|
result = client._send("prompt", [b"a"], enable_thinking=True)
|
||||||
|
|
||||||
|
self.assertEqual(result, GenAIResponse("", "still thinking"))
|
||||||
|
|
||||||
def _validated_client(self, server_context_size, provider_options=None):
|
def _validated_client(self, server_context_size, provider_options=None):
|
||||||
"""Build a client as if the server reported the given context size."""
|
"""Build a client as if the server reported the given context size."""
|
||||||
cfg = GenAIConfig(
|
cfg = GenAIConfig(
|
||||||
@@ -1093,6 +1168,98 @@ class TestImageTokenEstimate(unittest.TestCase):
|
|||||||
self.assertEqual(client._image_token_cache, {})
|
self.assertEqual(client._image_token_cache, {})
|
||||||
|
|
||||||
|
|
||||||
|
class _StubThinkingClient(GenAIClient):
|
||||||
|
def __init__(self, response: GenAIResponse | None) -> None:
|
||||||
|
super().__init__(GenAIConfig(provider="ollama", model="m"))
|
||||||
|
self.response = response
|
||||||
|
self.send_kwargs: dict = {}
|
||||||
|
|
||||||
|
@property
|
||||||
|
def supports_toggleable_thinking(self) -> bool:
|
||||||
|
return True
|
||||||
|
|
||||||
|
def _send(self, prompt, images, response_format=None, **kwargs):
|
||||||
|
self.send_kwargs = kwargs
|
||||||
|
return self.response
|
||||||
|
|
||||||
|
|
||||||
|
class TestReviewDescriptionThinking(unittest.TestCase):
|
||||||
|
REVIEW_ID = "1700000000.0-abc"
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(tmp.cleanup)
|
||||||
|
self.clips_dir = tmp.name
|
||||||
|
self.request_dir = os.path.join(
|
||||||
|
self.clips_dir, "genai-requests", self.REVIEW_ID
|
||||||
|
)
|
||||||
|
os.makedirs(self.request_dir)
|
||||||
|
patcher = patch("frigate.genai.CLIPS_DIR", self.clips_dir)
|
||||||
|
patcher.start()
|
||||||
|
self.addCleanup(patcher.stop)
|
||||||
|
|
||||||
|
def _generate(self, client, enable_thinking):
|
||||||
|
review_data = {
|
||||||
|
"id": self.REVIEW_ID,
|
||||||
|
"camera": "front",
|
||||||
|
"start": "Monday, 09:30 AM",
|
||||||
|
"duration": 10,
|
||||||
|
"zones": [],
|
||||||
|
"unified_objects": ["person"],
|
||||||
|
}
|
||||||
|
return client.generate_review_description(
|
||||||
|
review_data,
|
||||||
|
[b"a"],
|
||||||
|
[],
|
||||||
|
None,
|
||||||
|
True,
|
||||||
|
"activity context",
|
||||||
|
enable_thinking=enable_thinking,
|
||||||
|
)
|
||||||
|
|
||||||
|
def _saved(self, name):
|
||||||
|
path = os.path.join(self.request_dir, name)
|
||||||
|
if not os.path.exists(path):
|
||||||
|
return None
|
||||||
|
with open(path) as f:
|
||||||
|
return f.read()
|
||||||
|
|
||||||
|
def test_thinking_flag_reaches_provider_and_trace_is_saved(self):
|
||||||
|
answer = json.dumps(
|
||||||
|
{
|
||||||
|
"observations": [],
|
||||||
|
"scene": "A person walks up to the door.",
|
||||||
|
"title": "person at the door",
|
||||||
|
"shortSummary": "A person walks up to the door.",
|
||||||
|
"confidence": 0.9,
|
||||||
|
"potential_threat_level": 0,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
client = _StubThinkingClient(GenAIResponse(answer, "frame 1 shows a person"))
|
||||||
|
|
||||||
|
metadata = self._generate(client, enable_thinking=True)
|
||||||
|
|
||||||
|
self.assertTrue(client.send_kwargs["enable_thinking"])
|
||||||
|
self.assertEqual(metadata.title, "Person at the door")
|
||||||
|
self.assertEqual(self._saved("response.txt"), answer)
|
||||||
|
self.assertEqual(self._saved("reasoning.txt"), "frame 1 shows a person")
|
||||||
|
|
||||||
|
def test_trace_is_saved_when_answer_is_blank(self):
|
||||||
|
client = _StubThinkingClient(GenAIResponse("", "ran out of tokens"))
|
||||||
|
|
||||||
|
self.assertIsNone(self._generate(client, enable_thinking=True))
|
||||||
|
self.assertIsNone(self._saved("response.txt"))
|
||||||
|
self.assertEqual(self._saved("reasoning.txt"), "ran out of tokens")
|
||||||
|
|
||||||
|
def test_thinking_off_by_default(self):
|
||||||
|
client = _StubThinkingClient(None)
|
||||||
|
|
||||||
|
self._generate(client, enable_thinking=False)
|
||||||
|
|
||||||
|
self.assertFalse(client.send_kwargs["enable_thinking"])
|
||||||
|
self.assertIsNone(self._saved("reasoning.txt"))
|
||||||
|
|
||||||
|
|
||||||
class TestBaseClientTranscribe(unittest.TestCase):
|
class TestBaseClientTranscribe(unittest.TestCase):
|
||||||
"""Providers that don't implement the role must be inert, not broken."""
|
"""Providers that don't implement the role must be inert, not broken."""
|
||||||
|
|
||||||
|
|||||||
@@ -79,8 +79,24 @@ class TestEventNames(unittest.TestCase):
|
|||||||
self.assertEqual(event_name({"label": "animal"}), "an animal")
|
self.assertEqual(event_name({"label": "animal"}), "an animal")
|
||||||
|
|
||||||
def test_sub_labeled_objects_use_their_name(self):
|
def test_sub_labeled_objects_use_their_name(self):
|
||||||
event = {"label": "waste_bin", "sub_label": "Compost"}
|
self.assertEqual(
|
||||||
self.assertEqual(event_name(event), 'waste bin "Compost"')
|
event_name({"label": "waste_bin", "sub_label": "Compost"}),
|
||||||
|
"Compost",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
event_name({"label": "person", "sub_label": "nick"}),
|
||||||
|
"Nick",
|
||||||
|
)
|
||||||
|
self.assertEqual(
|
||||||
|
event_name({"label": "person", "sub_label": "nick_smith"}),
|
||||||
|
"Nick Smith",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_unknown_face_stays_unnamed(self):
|
||||||
|
self.assertEqual(
|
||||||
|
event_name({"label": "person", "sub_label": "unknown"}),
|
||||||
|
"a person",
|
||||||
|
)
|
||||||
|
|
||||||
def test_repeated_objects_are_never_numbered(self):
|
def test_repeated_objects_are_never_numbered(self):
|
||||||
# Whether these are the same subject is unknown, so the notes must not
|
# Whether these are the same subject is unknown, so the notes must not
|
||||||
@@ -184,8 +200,7 @@ class TestArrival(unittest.TestCase):
|
|||||||
phrases,
|
phrases,
|
||||||
[
|
[
|
||||||
"a person first detected at the right of the frame, moving up and left",
|
"a person first detected at the right of the frame, moving up and left",
|
||||||
'waste bin "Compost" first detected at the right of the frame, '
|
"Compost first detected at the right of the frame, moving up and left",
|
||||||
"moving up and left",
|
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -201,12 +216,12 @@ class TestArrival(unittest.TestCase):
|
|||||||
timeline = build_timeline([bin_], 100.0)
|
timeline = build_timeline([bin_], 100.0)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
timeline[0],
|
timeline[0],
|
||||||
(0.3, 'waste bin "Compost" first detected at the right of the frame'),
|
(0.3, "Compost first detected at the right of the frame"),
|
||||||
)
|
)
|
||||||
self.assertAlmostEqual(timeline[1][0], 21.4)
|
self.assertAlmostEqual(timeline[1][0], 21.4)
|
||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
timeline[1][1],
|
timeline[1][1],
|
||||||
'waste bin "Compost" starts moving up and left from the right of the frame',
|
"Compost starts moving up and left from the right of the frame",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
@@ -233,8 +248,8 @@ class TestStateChanges(unittest.TestCase):
|
|||||||
self.assertEqual(
|
self.assertEqual(
|
||||||
timeline[1:],
|
timeline[1:],
|
||||||
[
|
[
|
||||||
(20.0, 'waste bin "Compost" has stopped moving'),
|
(20.0, "Compost has stopped moving"),
|
||||||
(30.0, 'waste bin "Compost" starts moving again'),
|
(30.0, "Compost starts moving again"),
|
||||||
],
|
],
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -355,7 +370,10 @@ class TestClassificationChangeCaptions(unittest.TestCase):
|
|||||||
captions[2],
|
captions[2],
|
||||||
"Frame 3 of 3 (+20.0s):\n[state] front gate changed from closed to open",
|
"Frame 3 of 3 (+20.0s):\n[state] front gate changed from closed to open",
|
||||||
)
|
)
|
||||||
self.assertTrue(captions[0].splitlines()[1].startswith("[tracker] "))
|
self.assertTrue(
|
||||||
|
captions[0].splitlines()[1].startswith("a person first detected")
|
||||||
|
)
|
||||||
|
self.assertNotIn("[tracker]", captions[0])
|
||||||
|
|
||||||
def test_change_is_noted_without_tracked_objects(self):
|
def test_change_is_noted_without_tracked_objects(self):
|
||||||
with patch(
|
with patch(
|
||||||
|
|||||||
@@ -15,8 +15,10 @@ Setup:
|
|||||||
The script presents a menu to edit provider settings (provider, base URL,
|
The script presents a menu to edit provider settings (provider, base URL,
|
||||||
optional API key, model) or run an example. Examples are selected with the
|
optional API key, model) or run an example. Examples are selected with the
|
||||||
up/down arrow keys and launched with Enter. A writing style preset can be
|
up/down arrow keys and launched with Enter. A writing style preset can be
|
||||||
applied on top of the saved prompt to compare tone between runs, and after a
|
applied on top of the saved prompt to compare tone between runs, thinking can
|
||||||
response the model can be asked follow-up questions about the same frames.
|
be turned on for models with a per-request toggle (the reasoning trace is
|
||||||
|
printed before the response), and after a response the model can be asked
|
||||||
|
follow-up questions about the same frames.
|
||||||
|
|
||||||
Provider settings are stored in
|
Provider settings are stored in
|
||||||
testing-scripts/genai-review-examples/.settings.json (gitignored).
|
testing-scripts/genai-review-examples/.settings.json (gitignored).
|
||||||
@@ -311,11 +313,12 @@ def followup_loop(client: Any, base_prompt: str, frames: list[bytes]) -> None:
|
|||||||
f"Question: {question}"
|
f"Question: {question}"
|
||||||
)
|
)
|
||||||
start = time.monotonic()
|
start = time.monotonic()
|
||||||
answer = client._send(prompt, frames)
|
result = client._send(prompt, frames)
|
||||||
elapsed = time.monotonic() - start
|
elapsed = time.monotonic() - start
|
||||||
if answer is None:
|
if result is None:
|
||||||
print("No response from provider (see logs above).")
|
print("No response from provider (see logs above).")
|
||||||
continue
|
continue
|
||||||
|
answer = result.content
|
||||||
print(f"\n{answer}\n({elapsed:.1f}s)")
|
print(f"\n{answer}\n({elapsed:.1f}s)")
|
||||||
transcript = f"{prompt}\n\nYour answer:\n{answer}"
|
transcript = f"{prompt}\n\nYour answer:\n{answer}"
|
||||||
|
|
||||||
@@ -343,6 +346,11 @@ def run_example(settings: TesterSettings) -> None:
|
|||||||
return
|
return
|
||||||
style = styles[style_index]
|
style = styles[style_index]
|
||||||
|
|
||||||
|
thinking_index = select_option("Thinking:", ["off", "on"])
|
||||||
|
if thinking_index is None:
|
||||||
|
return
|
||||||
|
enable_thinking = thinking_index == 1
|
||||||
|
|
||||||
prompt_file = example / "prompt.txt"
|
prompt_file = example / "prompt.txt"
|
||||||
if not prompt_file.is_file():
|
if not prompt_file.is_file():
|
||||||
print(f"{example.name} has no prompt.txt, cannot run")
|
print(f"{example.name} has no prompt.txt, cannot run")
|
||||||
@@ -363,18 +371,26 @@ def run_example(settings: TesterSettings) -> None:
|
|||||||
concerns = ["_"] if "other_concerns" in prompt else []
|
concerns = ["_"] if "other_concerns" in prompt else []
|
||||||
response_format = build_review_description_response_format(concerns)
|
response_format = build_review_description_response_format(concerns)
|
||||||
|
|
||||||
|
if enable_thinking and not client.supports_toggleable_thinking:
|
||||||
|
print(" Warning: model has no per-request thinking toggle, sending anyway")
|
||||||
|
|
||||||
print(
|
print(
|
||||||
f"\nSending {len(frames)} frames to {settings.provider} "
|
f"\nSending {len(frames)} frames to {settings.provider} "
|
||||||
f"(model={settings.model or '(none)'}, style={style})..."
|
f"(model={settings.model or '(none)'}, style={style}, "
|
||||||
|
f"thinking={'on' if enable_thinking else 'off'})..."
|
||||||
)
|
)
|
||||||
start = time.monotonic()
|
start = time.monotonic()
|
||||||
response = client._send(prompt, frames, response_format)
|
result = client._send(prompt, frames, response_format, enable_thinking)
|
||||||
elapsed = time.monotonic() - start
|
elapsed = time.monotonic() - start
|
||||||
|
|
||||||
if response is None:
|
if result is None:
|
||||||
print("No response from provider (see logs above).")
|
print("No response from provider (see logs above).")
|
||||||
return
|
return
|
||||||
|
|
||||||
|
if result.reasoning:
|
||||||
|
print(f"\nReasoning:\n\n{result.reasoning}")
|
||||||
|
|
||||||
|
response = result.content
|
||||||
print(f"\nResponse ({elapsed:.1f}s):\n")
|
print(f"\nResponse ({elapsed:.1f}s):\n")
|
||||||
pretty_print_response(response)
|
pretty_print_response(response)
|
||||||
followup_loop(client, f"{prompt}\n\nYour analysis:\n{response}", frames)
|
followup_loop(client, f"{prompt}\n\nYour analysis:\n{response}", frames)
|
||||||
|
|||||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -0,0 +1,90 @@
|
|||||||
|
/**
|
||||||
|
* Review GenAI thinking toggle tests -- MEDIUM tier.
|
||||||
|
*
|
||||||
|
* Thinking can only be switched per request by providers that expose a
|
||||||
|
* toggle, so the option is shown only when one of those providers holds the
|
||||||
|
* descriptions role.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import { readFileSync } from "node:fs";
|
||||||
|
import { resolve, dirname } from "node:path";
|
||||||
|
import { fileURLToPath } from "node:url";
|
||||||
|
import { test, expect } from "../../fixtures/frigate-test";
|
||||||
|
import type { Page } from "@playwright/test";
|
||||||
|
import { configFactory } from "../../fixtures/mock-data/config";
|
||||||
|
|
||||||
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||||
|
const CONFIG_SCHEMA = JSON.parse(
|
||||||
|
readFileSync(
|
||||||
|
resolve(__dirname, "../../fixtures/mock-data/config-schema.json"),
|
||||||
|
"utf-8",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const SETTINGS_URL = "/settings?page=globalReview";
|
||||||
|
|
||||||
|
async function installRoutes(
|
||||||
|
page: Page,
|
||||||
|
genai: Record<string, { provider: string; model: string; roles: string[] }>,
|
||||||
|
) {
|
||||||
|
const config = configFactory({ genai } as never);
|
||||||
|
|
||||||
|
await page.route("**/api/config/schema.json", (route) =>
|
||||||
|
route.fulfill({ json: CONFIG_SCHEMA }),
|
||||||
|
);
|
||||||
|
await page.route("**/api/config", (route) =>
|
||||||
|
route.request().method() === "GET"
|
||||||
|
? route.fulfill({ json: config })
|
||||||
|
: route.fulfill({ json: { success: true } }),
|
||||||
|
);
|
||||||
|
await page.route("**/api/config/raw_paths", (route) =>
|
||||||
|
route.fulfill({ json: {} }),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function openGenAIConfig(page: Page) {
|
||||||
|
await page.getByText("GenAI config", { exact: true }).click();
|
||||||
|
await expect(page.locator("#pageRoot")).toContainText("Response style");
|
||||||
|
}
|
||||||
|
|
||||||
|
test.describe("review genai thinking @medium", () => {
|
||||||
|
test("is shown when the descriptions provider has a thinking toggle", async ({
|
||||||
|
frigateApp,
|
||||||
|
}) => {
|
||||||
|
await installRoutes(frigateApp.page, {
|
||||||
|
local: { provider: "llamacpp", model: "m", roles: ["descriptions"] },
|
||||||
|
});
|
||||||
|
await frigateApp.goto(SETTINGS_URL);
|
||||||
|
await openGenAIConfig(frigateApp.page);
|
||||||
|
|
||||||
|
await expect(frigateApp.page.locator("#pageRoot")).toContainText(
|
||||||
|
"Enable thinking",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("is hidden when the descriptions provider has no thinking toggle", async ({
|
||||||
|
frigateApp,
|
||||||
|
}) => {
|
||||||
|
// the chat provider supports the toggle, but descriptions go elsewhere
|
||||||
|
await installRoutes(frigateApp.page, {
|
||||||
|
local: { provider: "ollama", model: "m", roles: ["chat"] },
|
||||||
|
cloud: { provider: "gemini", model: "m", roles: ["descriptions"] },
|
||||||
|
});
|
||||||
|
await frigateApp.goto(SETTINGS_URL);
|
||||||
|
await openGenAIConfig(frigateApp.page);
|
||||||
|
|
||||||
|
await expect(frigateApp.page.locator("#pageRoot")).not.toContainText(
|
||||||
|
"Enable thinking",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("is hidden when no provider is configured", async ({ frigateApp }) => {
|
||||||
|
await installRoutes(frigateApp.page, {});
|
||||||
|
await frigateApp.goto(SETTINGS_URL);
|
||||||
|
await openGenAIConfig(frigateApp.page);
|
||||||
|
|
||||||
|
await expect(frigateApp.page.locator("#pageRoot")).not.toContainText(
|
||||||
|
"Enable thinking",
|
||||||
|
);
|
||||||
|
});
|
||||||
|
});
|
||||||
@@ -689,6 +689,10 @@
|
|||||||
"label": "Response style",
|
"label": "Response style",
|
||||||
"description": "Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged."
|
"description": "Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged."
|
||||||
},
|
},
|
||||||
|
"enable_thinking": {
|
||||||
|
"label": "Enable thinking",
|
||||||
|
"description": "Let the model reason before writing the review description. Only applies to models with a per-request thinking toggle (Ollama and llama.cpp providers); other models keep their default behavior. Thinking increases response time and token usage. When thumbnail saving is enabled, the reasoning is saved alongside the response."
|
||||||
|
},
|
||||||
"activity_context_prompt": {
|
"activity_context_prompt": {
|
||||||
"label": "Activity context prompt",
|
"label": "Activity context prompt",
|
||||||
"description": "Custom prompt describing what is and is not suspicious activity to provide context for GenAI summaries."
|
"description": "Custom prompt describing what is and is not suspicious activity to provide context for GenAI summaries."
|
||||||
|
|||||||
@@ -1060,6 +1060,10 @@
|
|||||||
"label": "Response style",
|
"label": "Response style",
|
||||||
"description": "Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged."
|
"description": "Writing style preset for generated review descriptions. Presets adjust the tone and level of detail of the user-facing title, summary, and scene description; 'default' leaves the built-in prompt unchanged."
|
||||||
},
|
},
|
||||||
|
"enable_thinking": {
|
||||||
|
"label": "Enable thinking",
|
||||||
|
"description": "Let the model reason before writing the review description. Only applies to models with a per-request thinking toggle (Ollama and llama.cpp providers); other models keep their default behavior. Thinking increases response time and token usage. When thumbnail saving is enabled, the reasoning is saved alongside the response."
|
||||||
|
},
|
||||||
"activity_context_prompt": {
|
"activity_context_prompt": {
|
||||||
"label": "Activity context prompt",
|
"label": "Activity context prompt",
|
||||||
"description": "Custom prompt describing what is and is not suspicious activity to provide context for GenAI summaries."
|
"description": "Custom prompt describing what is and is not suspicious activity to provide context for GenAI summaries."
|
||||||
|
|||||||
@@ -1,5 +1,19 @@
|
|||||||
|
import type { HiddenFieldContext } from "@/types/configForm";
|
||||||
import type { SectionConfigOverrides } from "./types";
|
import type { SectionConfigOverrides } from "./types";
|
||||||
|
|
||||||
|
// Only these providers expose a per-request thinking toggle, so the option
|
||||||
|
// has no effect when any other provider holds the descriptions role
|
||||||
|
const THINKING_TOGGLE_PROVIDERS = new Set(["ollama", "llamacpp"]);
|
||||||
|
|
||||||
|
const hideThinkingWithoutToggle = (ctx: HiddenFieldContext): string[] => {
|
||||||
|
const provider = Object.values(ctx.fullConfig.genai ?? {}).find((agent) =>
|
||||||
|
agent.roles?.includes("descriptions"),
|
||||||
|
)?.provider;
|
||||||
|
return provider && THINKING_TOGGLE_PROVIDERS.has(provider)
|
||||||
|
? []
|
||||||
|
: ["genai.enable_thinking"];
|
||||||
|
};
|
||||||
|
|
||||||
const review: SectionConfigOverrides = {
|
const review: SectionConfigOverrides = {
|
||||||
base: {
|
base: {
|
||||||
sectionDocs: "/configuration/review",
|
sectionDocs: "/configuration/review",
|
||||||
@@ -98,6 +112,7 @@ const review: SectionConfigOverrides = {
|
|||||||
"genai.frame_mode": "/configuration/genai/genai_review#frame-mode",
|
"genai.frame_mode": "/configuration/genai/genai_review#frame-mode",
|
||||||
"genai.additional_concerns":
|
"genai.additional_concerns":
|
||||||
"/configuration/genai/genai_review#additional-concerns",
|
"/configuration/genai/genai_review#additional-concerns",
|
||||||
|
"genai.enable_thinking": "/configuration/genai/genai_review#thinking",
|
||||||
},
|
},
|
||||||
restartRequired: [],
|
restartRequired: [],
|
||||||
fieldOrder: ["alerts", "detections", "genai", "genai.enabled"],
|
fieldOrder: ["alerts", "detections", "genai", "genai.enabled"],
|
||||||
@@ -107,6 +122,7 @@ const review: SectionConfigOverrides = {
|
|||||||
"alerts.enabled_in_config",
|
"alerts.enabled_in_config",
|
||||||
"detections.enabled_in_config",
|
"detections.enabled_in_config",
|
||||||
"genai.enabled_in_config",
|
"genai.enabled_in_config",
|
||||||
|
hideThinkingWithoutToggle,
|
||||||
],
|
],
|
||||||
advancedFields: [],
|
advancedFields: [],
|
||||||
uiSchema: {
|
uiSchema: {
|
||||||
|
|||||||
Reference in New Issue
Block a user