mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-07-21 03:09:02 +03:00
Compare commits
13
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cf9a4a9407 | ||
|
|
06e5852743 | ||
|
|
b0f9fefd97 | ||
|
|
c2203ead61 | ||
|
|
1918e9682b | ||
|
|
fefb264e55 | ||
|
|
63b2384d87 | ||
|
|
d76335096a | ||
|
|
ab3ded38e6 | ||
|
|
2376bcaf97 | ||
|
|
6a27c47808 | ||
|
|
7703cfbfee | ||
|
|
25e8c2a051 |
@@ -5,7 +5,7 @@ title: Enrichments
|
||||
|
||||
# Enrichments
|
||||
|
||||
Some of Frigate's enrichments can use a discrete GPU / NPU for accelerated processing.
|
||||
Some of Frigate's enrichments can use a discrete GPU or integrated GPU for accelerated processing.
|
||||
|
||||
## Requirements
|
||||
|
||||
@@ -18,8 +18,10 @@ Object detection and enrichments (like Semantic Search, Face Recognition, and Li
|
||||
- **Intel**
|
||||
|
||||
- OpenVINO will automatically be detected and used for enrichments in the default Frigate image.
|
||||
- **Note:** Intel NPUs have limited model support for enrichments. GPU is recommended for enrichments when available.
|
||||
|
||||
- **Nvidia**
|
||||
|
||||
- Nvidia GPUs will automatically be detected and used for enrichments in the `-tensorrt` Frigate image.
|
||||
- Jetson devices will automatically be detected and used for enrichments in the `-tensorrt-jp6` Frigate image.
|
||||
|
||||
|
||||
@@ -261,6 +261,8 @@ OpenVINO is supported on 6th Gen Intel platforms (Skylake) and newer. It will al
|
||||
|
||||
:::tip
|
||||
|
||||
**NPU + GPU Systems:** If you have both NPU and GPU available (Intel Core Ultra processors), use NPU for object detection and GPU for enrichments (semantic search, face recognition, etc.) for best performance and compatibility.
|
||||
|
||||
When using many cameras one detector may not be enough to keep up. Multiple detectors can be defined assuming GPU resources are available. An example configuration would be:
|
||||
|
||||
```yaml
|
||||
@@ -283,7 +285,7 @@ detectors:
|
||||
| [RF-DETR](#rf-detr) | ✅ | ✅ | Requires XE iGPU or Arc |
|
||||
| [YOLO-NAS](#yolo-nas) | ✅ | ✅ | |
|
||||
| [MobileNet v2](#ssdlite-mobilenet-v2) | ✅ | ✅ | Fast and lightweight model, less accurate than larger models |
|
||||
| [YOLOX](#yolox) | ✅ | ? | |
|
||||
| [YOLOX](#yolox) | ✅ | ? | |
|
||||
| [D-FINE](#d-fine) | ❌ | ❌ | |
|
||||
|
||||
#### SSDLite MobileNet v2
|
||||
|
||||
@@ -78,7 +78,7 @@ Switching between V1 and V2 requires reindexing your embeddings. The embeddings
|
||||
|
||||
### GPU Acceleration
|
||||
|
||||
The CLIP models are downloaded in ONNX format, and the `large` model can be accelerated using GPU / NPU hardware, when available. This depends on the Docker build that is used. You can also target a specific device in a multi-GPU installation.
|
||||
The CLIP models are downloaded in ONNX format, and the `large` model can be accelerated using GPU hardware, when available. This depends on the Docker build that is used. You can also target a specific device in a multi-GPU installation.
|
||||
|
||||
```yaml
|
||||
semantic_search:
|
||||
@@ -90,7 +90,7 @@ semantic_search:
|
||||
|
||||
:::info
|
||||
|
||||
If the correct build is used for your GPU / NPU and the `large` model is configured, then the GPU / NPU will be detected and used automatically.
|
||||
If the correct build is used for your GPU / NPU and the `large` model is configured, then the GPU will be detected and used automatically.
|
||||
Specify the `device` option to target a specific GPU in a multi-GPU system (see [onnxruntime's provider options](https://onnxruntime.ai/docs/execution-providers/)).
|
||||
If you do not specify a device, the first available GPU will be used.
|
||||
|
||||
|
||||
@@ -418,8 +418,8 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi):
|
||||
obj_data["box"][2],
|
||||
obj_data["box"][3],
|
||||
max(
|
||||
obj_data["box"][1] - obj_data["box"][0],
|
||||
obj_data["box"][3] - obj_data["box"][2],
|
||||
obj_data["box"][2] - obj_data["box"][0],
|
||||
obj_data["box"][3] - obj_data["box"][1],
|
||||
),
|
||||
1.0,
|
||||
)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
import logging
|
||||
import os
|
||||
import platform
|
||||
import threading
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any
|
||||
|
||||
@@ -161,12 +162,12 @@ class CudaGraphRunner(BaseModelRunner):
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def is_complex_model(model_type: str) -> bool:
|
||||
def is_model_supported(model_type: str) -> bool:
|
||||
# Import here to avoid circular imports
|
||||
from frigate.detectors.detector_config import ModelTypeEnum
|
||||
from frigate.embeddings.types import EnrichmentModelTypeEnum
|
||||
|
||||
return model_type in [
|
||||
return model_type not in [
|
||||
ModelTypeEnum.yolonas.value,
|
||||
EnrichmentModelTypeEnum.paddleocr.value,
|
||||
EnrichmentModelTypeEnum.jina_v1.value,
|
||||
@@ -239,9 +240,30 @@ class OpenVINOModelRunner(BaseModelRunner):
|
||||
EnrichmentModelTypeEnum.jina_v2.value,
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def is_model_npu_supported(model_type: str) -> bool:
|
||||
# Import here to avoid circular imports
|
||||
from frigate.embeddings.types import EnrichmentModelTypeEnum
|
||||
|
||||
return model_type not in [
|
||||
EnrichmentModelTypeEnum.paddleocr.value,
|
||||
EnrichmentModelTypeEnum.jina_v1.value,
|
||||
EnrichmentModelTypeEnum.jina_v2.value,
|
||||
EnrichmentModelTypeEnum.arcface.value,
|
||||
]
|
||||
|
||||
def __init__(self, model_path: str, device: str, model_type: str, **kwargs):
|
||||
self.model_path = model_path
|
||||
self.device = device
|
||||
|
||||
if device == "NPU" and not OpenVINOModelRunner.is_model_npu_supported(
|
||||
model_type
|
||||
):
|
||||
logger.warning(
|
||||
f"OpenVINO model {model_type} is not supported on NPU, using GPU instead"
|
||||
)
|
||||
device = "GPU"
|
||||
|
||||
self.complex_model = OpenVINOModelRunner.is_complex_model(model_type)
|
||||
|
||||
if not os.path.isfile(model_path):
|
||||
@@ -269,6 +291,10 @@ class OpenVINOModelRunner(BaseModelRunner):
|
||||
self.infer_request = self.compiled_model.create_infer_request()
|
||||
self.input_tensor: ov.Tensor | None = None
|
||||
|
||||
# Thread lock to prevent concurrent inference (needed for JinaV2 which shares
|
||||
# one runner between text and vision embeddings called from different threads)
|
||||
self._inference_lock = threading.Lock()
|
||||
|
||||
if not self.complex_model:
|
||||
try:
|
||||
input_shape = self.compiled_model.inputs[0].get_shape()
|
||||
@@ -312,67 +338,70 @@ class OpenVINOModelRunner(BaseModelRunner):
|
||||
Returns:
|
||||
List of output tensors
|
||||
"""
|
||||
# Handle single input case for backward compatibility
|
||||
if (
|
||||
len(inputs) == 1
|
||||
and len(self.compiled_model.inputs) == 1
|
||||
and self.input_tensor is not None
|
||||
):
|
||||
# Single input case - use the pre-allocated tensor for efficiency
|
||||
input_data = list(inputs.values())[0]
|
||||
np.copyto(self.input_tensor.data, input_data)
|
||||
self.infer_request.infer(self.input_tensor)
|
||||
else:
|
||||
if self.complex_model:
|
||||
try:
|
||||
# This ensures the model starts with a clean state for each sequence
|
||||
# Important for RNN models like PaddleOCR recognition
|
||||
self.infer_request.reset_state()
|
||||
except Exception:
|
||||
# this will raise an exception for models with AUTO set as the device
|
||||
pass
|
||||
# Lock prevents concurrent access to infer_request
|
||||
# Needed for JinaV2: genai thread (text) + embeddings thread (vision)
|
||||
with self._inference_lock:
|
||||
# Handle single input case for backward compatibility
|
||||
if (
|
||||
len(inputs) == 1
|
||||
and len(self.compiled_model.inputs) == 1
|
||||
and self.input_tensor is not None
|
||||
):
|
||||
# Single input case - use the pre-allocated tensor for efficiency
|
||||
input_data = list(inputs.values())[0]
|
||||
np.copyto(self.input_tensor.data, input_data)
|
||||
self.infer_request.infer(self.input_tensor)
|
||||
else:
|
||||
if self.complex_model:
|
||||
try:
|
||||
# This ensures the model starts with a clean state for each sequence
|
||||
# Important for RNN models like PaddleOCR recognition
|
||||
self.infer_request.reset_state()
|
||||
except Exception:
|
||||
# this will raise an exception for models with AUTO set as the device
|
||||
pass
|
||||
|
||||
# Multiple inputs case - set each input by name
|
||||
for input_name, input_data in inputs.items():
|
||||
# Find the input by name and its index
|
||||
input_port = None
|
||||
input_index = None
|
||||
for idx, port in enumerate(self.compiled_model.inputs):
|
||||
if port.get_any_name() == input_name:
|
||||
input_port = port
|
||||
input_index = idx
|
||||
break
|
||||
# Multiple inputs case - set each input by name
|
||||
for input_name, input_data in inputs.items():
|
||||
# Find the input by name and its index
|
||||
input_port = None
|
||||
input_index = None
|
||||
for idx, port in enumerate(self.compiled_model.inputs):
|
||||
if port.get_any_name() == input_name:
|
||||
input_port = port
|
||||
input_index = idx
|
||||
break
|
||||
|
||||
if input_port is None:
|
||||
raise ValueError(f"Input '{input_name}' not found in model")
|
||||
if input_port is None:
|
||||
raise ValueError(f"Input '{input_name}' not found in model")
|
||||
|
||||
# Create tensor with the correct element type
|
||||
input_element_type = input_port.get_element_type()
|
||||
# Create tensor with the correct element type
|
||||
input_element_type = input_port.get_element_type()
|
||||
|
||||
# Ensure input data matches the expected dtype to prevent type mismatches
|
||||
# that can occur with models like Jina-CLIP v2 running on OpenVINO
|
||||
expected_dtype = input_element_type.to_dtype()
|
||||
if input_data.dtype != expected_dtype:
|
||||
logger.debug(
|
||||
f"Converting input '{input_name}' from {input_data.dtype} to {expected_dtype}"
|
||||
)
|
||||
input_data = input_data.astype(expected_dtype)
|
||||
# Ensure input data matches the expected dtype to prevent type mismatches
|
||||
# that can occur with models like Jina-CLIP v2 running on OpenVINO
|
||||
expected_dtype = input_element_type.to_dtype()
|
||||
if input_data.dtype != expected_dtype:
|
||||
logger.debug(
|
||||
f"Converting input '{input_name}' from {input_data.dtype} to {expected_dtype}"
|
||||
)
|
||||
input_data = input_data.astype(expected_dtype)
|
||||
|
||||
input_tensor = ov.Tensor(input_element_type, input_data.shape)
|
||||
np.copyto(input_tensor.data, input_data)
|
||||
input_tensor = ov.Tensor(input_element_type, input_data.shape)
|
||||
np.copyto(input_tensor.data, input_data)
|
||||
|
||||
# Set the input tensor for the specific port index
|
||||
self.infer_request.set_input_tensor(input_index, input_tensor)
|
||||
# Set the input tensor for the specific port index
|
||||
self.infer_request.set_input_tensor(input_index, input_tensor)
|
||||
|
||||
# Run inference
|
||||
self.infer_request.infer()
|
||||
# Run inference
|
||||
self.infer_request.infer()
|
||||
|
||||
# Get all output tensors
|
||||
outputs = []
|
||||
for i in range(len(self.compiled_model.outputs)):
|
||||
outputs.append(self.infer_request.get_output_tensor(i).data)
|
||||
# Get all output tensors
|
||||
outputs = []
|
||||
for i in range(len(self.compiled_model.outputs)):
|
||||
outputs.append(self.infer_request.get_output_tensor(i).data)
|
||||
|
||||
return outputs
|
||||
return outputs
|
||||
|
||||
|
||||
class RKNNModelRunner(BaseModelRunner):
|
||||
@@ -500,7 +529,7 @@ def get_optimized_runner(
|
||||
return OpenVINOModelRunner(model_path, device, model_type, **kwargs)
|
||||
|
||||
if (
|
||||
not CudaGraphRunner.is_complex_model(model_type)
|
||||
not CudaGraphRunner.is_model_supported(model_type)
|
||||
and providers[0] == "CUDAExecutionProvider"
|
||||
):
|
||||
options[0] = {
|
||||
|
||||
@@ -55,29 +55,32 @@ export default function DetailActionsMenu({
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuPortal>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuItem>
|
||||
<a
|
||||
className="w-full"
|
||||
href={`${baseUrl}api/events/${search.id}/snapshot.jpg?bbox=1`}
|
||||
download={`${search.camera}_${search.label}.jpg`}
|
||||
>
|
||||
<div className="flex cursor-pointer items-center gap-2">
|
||||
<span>{t("itemMenu.downloadSnapshot.label")}</span>
|
||||
</div>
|
||||
</a>
|
||||
</DropdownMenuItem>
|
||||
|
||||
<DropdownMenuItem>
|
||||
<a
|
||||
className="w-full"
|
||||
href={`${baseUrl}api/${search.camera}/${clipTimeRange}/clip.mp4`}
|
||||
download
|
||||
>
|
||||
<div className="flex cursor-pointer items-center gap-2">
|
||||
<span>{t("itemMenu.downloadVideo.label")}</span>
|
||||
</div>
|
||||
</a>
|
||||
</DropdownMenuItem>
|
||||
{search.has_snapshot && (
|
||||
<DropdownMenuItem>
|
||||
<a
|
||||
className="w-full"
|
||||
href={`${baseUrl}api/events/${search.id}/snapshot.jpg?bbox=1`}
|
||||
download={`${search.camera}_${search.label}.jpg`}
|
||||
>
|
||||
<div className="flex cursor-pointer items-center gap-2">
|
||||
<span>{t("itemMenu.downloadSnapshot.label")}</span>
|
||||
</div>
|
||||
</a>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
{search.has_clip && (
|
||||
<DropdownMenuItem>
|
||||
<a
|
||||
className="w-full"
|
||||
href={`${baseUrl}api/${search.camera}/${clipTimeRange}/clip.mp4`}
|
||||
download
|
||||
>
|
||||
<div className="flex cursor-pointer items-center gap-2">
|
||||
<span>{t("itemMenu.downloadVideo.label")}</span>
|
||||
</div>
|
||||
</a>
|
||||
</DropdownMenuItem>
|
||||
)}
|
||||
|
||||
{config?.semantic_search.enabled &&
|
||||
setSimilarity != undefined &&
|
||||
|
||||
@@ -306,7 +306,7 @@ function DialogContentComponent({
|
||||
if (page === "tracking_details") {
|
||||
return (
|
||||
<TrackingDetails
|
||||
className={cn("size-full", !isDesktop && "flex flex-col gap-4")}
|
||||
className={cn(isDesktop ? "size-full" : "flex flex-col gap-4")}
|
||||
event={search as unknown as Event}
|
||||
tabs={
|
||||
isDesktop ? (
|
||||
@@ -584,7 +584,7 @@ export default function SearchDetailDialog({
|
||||
"scrollbar-container overflow-y-auto",
|
||||
isDesktop &&
|
||||
"max-h-[95dvh] sm:max-w-xl md:max-w-4xl lg:max-w-[70%]",
|
||||
isMobile && "px-4",
|
||||
isMobile && "flex h-full flex-col px-4",
|
||||
)}
|
||||
onInteractOutside={(e) => {
|
||||
if (isPopoverOpen) {
|
||||
@@ -596,7 +596,7 @@ export default function SearchDetailDialog({
|
||||
}
|
||||
}}
|
||||
>
|
||||
<Header>
|
||||
<Header className={cn(!isDesktop && "top-0 z-[60] mb-0")}>
|
||||
<Title>{t("trackedObjectDetails")}</Title>
|
||||
<Description className="sr-only">
|
||||
{t("trackedObjectDetails")}
|
||||
@@ -1078,12 +1078,31 @@ function ObjectDetailsTab({
|
||||
});
|
||||
|
||||
setState("submitted");
|
||||
setSearch({
|
||||
...search,
|
||||
plus_id: "new_upload",
|
||||
});
|
||||
mutate(
|
||||
(key) =>
|
||||
typeof key === "string" &&
|
||||
(key.includes("events") ||
|
||||
key.includes("events/search") ||
|
||||
key.includes("events/explore")),
|
||||
(currentData: SearchResult[][] | SearchResult[] | undefined) => {
|
||||
if (!currentData) return currentData;
|
||||
// optimistic update
|
||||
return currentData
|
||||
.flat()
|
||||
.map((event) =>
|
||||
event.id === search.id
|
||||
? { ...event, plus_id: "new_upload" }
|
||||
: event,
|
||||
);
|
||||
},
|
||||
{
|
||||
optimisticData: true,
|
||||
rollbackOnError: true,
|
||||
revalidate: false,
|
||||
},
|
||||
);
|
||||
},
|
||||
[search, setSearch],
|
||||
[search, mutate],
|
||||
);
|
||||
|
||||
const popoverContainerRef = useRef<HTMLDivElement | null>(null);
|
||||
@@ -1243,8 +1262,8 @@ function ObjectDetailsTab({
|
||||
</div>
|
||||
|
||||
{search.data.type === "object" &&
|
||||
!search.plus_id &&
|
||||
config?.plus?.enabled && (
|
||||
config?.plus?.enabled &&
|
||||
search.has_snapshot && (
|
||||
<div
|
||||
className={cn(
|
||||
"my-2 flex w-full flex-col justify-between gap-1.5",
|
||||
|
||||
@@ -352,7 +352,8 @@ export function TrackingDetails({
|
||||
className={cn(
|
||||
isDesktop
|
||||
? "flex size-full justify-evenly gap-4 overflow-hidden"
|
||||
: "flex size-full flex-col gap-2",
|
||||
: "flex flex-col gap-2",
|
||||
!isDesktop && cameraAspect === "tall" && "size-full",
|
||||
className,
|
||||
)}
|
||||
>
|
||||
@@ -719,9 +720,13 @@ function LifecycleIconRow({
|
||||
backgroundColor: `rgb(${color})`,
|
||||
}}
|
||||
/>
|
||||
<span className="smart-capitalize">
|
||||
{item.data?.zones_friendly_names?.[zidx] ??
|
||||
zone.replaceAll("_", " ")}
|
||||
<span
|
||||
className={cn(
|
||||
item.data?.zones_friendly_names?.[zidx] === zone &&
|
||||
"smart-capitalize",
|
||||
)}
|
||||
>
|
||||
{item.data?.zones_friendly_names?.[zidx]}
|
||||
</span>
|
||||
</Badge>
|
||||
);
|
||||
|
||||
@@ -576,6 +576,7 @@ export default function ZoneEditPane({
|
||||
control={form.control}
|
||||
nameField="friendly_name"
|
||||
idField="name"
|
||||
idVisible={(polygon && polygon.name.length > 0) ?? false}
|
||||
nameLabel={t("masksAndZones.zones.name.title")}
|
||||
nameDescription={t("masksAndZones.zones.name.tips")}
|
||||
placeholderName={t("masksAndZones.zones.name.inputPlaceHolder")}
|
||||
|
||||
@@ -21,20 +21,30 @@ export const capitalizeAll = (text: string): string => {
|
||||
* @returns A valid camera identifier (lowercase, alphanumeric, max 8 chars)
|
||||
*/
|
||||
export function generateFixedHash(name: string, prefix: string = "id"): string {
|
||||
// Safely encode Unicode as UTF-8 bytes
|
||||
// Use the full UTF-8 bytes of the name and compute an FNV-1a 32-bit hash.
|
||||
// This is deterministic, fast, works with Unicode and avoids collisions from
|
||||
// simple truncation of base64 output.
|
||||
const utf8Bytes = new TextEncoder().encode(name);
|
||||
|
||||
// Convert to base64 manually
|
||||
let binary = "";
|
||||
for (const byte of utf8Bytes) {
|
||||
binary += String.fromCharCode(byte);
|
||||
// FNV-1a 32-bit hash algorithm
|
||||
let hash = 0x811c9dc5; // FNV offset basis
|
||||
for (let i = 0; i < utf8Bytes.length; i++) {
|
||||
hash ^= utf8Bytes[i];
|
||||
// Multiply by FNV prime (0x01000193) with 32-bit overflow
|
||||
hash = (hash >>> 0) * 0x01000193;
|
||||
// Ensure 32-bit unsigned integer
|
||||
hash >>>= 0;
|
||||
}
|
||||
const base64 = btoa(binary);
|
||||
|
||||
// Strip out non-alphanumeric characters and truncate
|
||||
const cleanHash = base64.replace(/[^a-zA-Z0-9]/g, "").substring(0, 8);
|
||||
// Convert to an 8-character lowercase hex string
|
||||
const hashHex = (hash >>> 0).toString(16).padStart(8, "0").toLowerCase();
|
||||
|
||||
return `${prefix}_${cleanHash.toLowerCase()}`;
|
||||
// Ensure the first character is a letter to avoid an identifier that's purely
|
||||
// numeric (isValidId forbids all-digit IDs). If it starts with a digit,
|
||||
// replace with 'a'. This is extremely unlikely but a simple safeguard.
|
||||
const safeHash = /^[0-9]/.test(hashHex[0]) ? `a${hashHex.slice(1)}` : hashHex;
|
||||
|
||||
return `${prefix}_${safeHash}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -98,12 +98,12 @@ export default function CameraSettingsView({
|
||||
return Object.entries(cameraConfig.zones).map(([name, zoneData]) => ({
|
||||
camera: cameraConfig.name,
|
||||
name,
|
||||
friendly_name: getZoneName(name, cameraConfig.name),
|
||||
friendly_name: cameraConfig.zones[name].friendly_name,
|
||||
objects: zoneData.objects,
|
||||
color: zoneData.color,
|
||||
}));
|
||||
}
|
||||
}, [cameraConfig, getZoneName]);
|
||||
}, [cameraConfig]);
|
||||
|
||||
const alertsLabels = useMemo(() => {
|
||||
return cameraConfig?.review.alerts.labels
|
||||
@@ -533,8 +533,14 @@ export default function CameraSettingsView({
|
||||
}}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormLabel className="font-normal smart-capitalize">
|
||||
{zone.friendly_name}
|
||||
<FormLabel
|
||||
className={cn(
|
||||
"font-normal",
|
||||
!zone.friendly_name &&
|
||||
"smart-capitalize",
|
||||
)}
|
||||
>
|
||||
{zone.friendly_name || zone.name}
|
||||
</FormLabel>
|
||||
</FormItem>
|
||||
)}
|
||||
@@ -632,8 +638,14 @@ export default function CameraSettingsView({
|
||||
}}
|
||||
/>
|
||||
</FormControl>
|
||||
<FormLabel className="font-normal smart-capitalize">
|
||||
{zone.friendly_name}
|
||||
<FormLabel
|
||||
className={cn(
|
||||
"font-normal",
|
||||
!zone.friendly_name &&
|
||||
"smart-capitalize",
|
||||
)}
|
||||
>
|
||||
{zone.friendly_name || zone.name}
|
||||
</FormLabel>
|
||||
</FormItem>
|
||||
)}
|
||||
|
||||
Reference in New Issue
Block a user