Refactor detector and model management (#23995)

* Refactor detector and model management

* Fix model resolution field

frigate/detectors/detector_config.py

@@ -3,7 +3,7 @@ import json
 import logging
 import os
 from enum import Enum
-from typing import Any
+from typing import Any, ClassVar
 
 import requests
 from pydantic import BaseModel, ConfigDict, Field
@@ -15,6 +15,9 @@ from frigate.util.builtin import generate_color_palette, load_labels
 
 logger = logging.getLogger(__name__)
 
+# attributes that are recognized rather than shown as a logo
+NON_LOGO_ATTRIBUTES = ["face", "license_plate"]
+
 
 class PixelFormatEnum(str, Enum):
     rgb = "rgb"
@@ -44,7 +47,27 @@ class ModelTypeEnum(str, Enum):
     yologeneric = "yolo-generic"
 
 
+class SceneEnum(str, Enum):
+    """The camera environment a detection model is intended for."""
+
+    all = "all"
+    indoor = "indoor"
+    outdoor = "outdoor"
+    indoor_thermal = "indoor_thermal"
+    outdoor_thermal = "outdoor_thermal"
+
+
 class ModelConfig(BaseModel):
+    scene: SceneEnum = Field(
+        default=SceneEnum.all,
+        title="Model scene",
+        description="The camera environment this model is used for. Cameras select a model by setting detect.scene to a matching value, and 'all' is used by any camera that does not set one.",
+    )
+    devices: list[str] = Field(
+        default_factory=list,
+        title="Detection hardware",
+        description="Hardware this model runs on, as '<detector>' or '<detector>:<device>' (for example 'edgetpu:pci:0' or 'openvino:GPU'). Listing the same device more than once runs additional inference processes on it.",
+    )
     path: str | None = Field(
         None,
         title="Custom object detector model path",
@@ -111,7 +134,7 @@ class ModelConfig(BaseModel):
 
     @property
     def non_logo_attributes(self) -> list[str]:
-        return ["face", "license_plate"]
+        return NON_LOGO_ATTRIBUTES
 
     @property
     def all_attributes(self) -> list[str]:
@@ -201,9 +224,7 @@ class ModelConfig(BaseModel):
             unique_attributes.update(attributes)
 
         self._all_attributes = list(unique_attributes)
-        self._all_attribute_logos = list(
-            unique_attributes - set(["face", "license_plate"])
-        )
+        self._all_attribute_logos = list(unique_attributes - set(NON_LOGO_ATTRIBUTES))
 
         self._merged_labelmap = {
             **{int(key): val for key, val in model_info["labelMap"].items()},
@@ -234,6 +255,14 @@ class ModelConfig(BaseModel):
 
 
 class BaseDetectorConfig(BaseModel):
+    # how the trailing part of a device string ("openvino:GPU" -> "GPU") maps onto
+    # this detector's fields, and whether the same device may be listed more than
+    # once to run additional inference processes against it. Most accelerators
+    # multiplex fine, so this is opt-out rather than opt-in.
+    device_spec_field: ClassVar[str] = "device"
+    device_spec_type: ClassVar[type] = str
+    shareable: ClassVar[bool] = True
+
     # the type field must be defined in all subclasses
     type: str = Field(
         default="cpu",

frigate/detectors/detector_types.py

@@ -2,7 +2,7 @@ import importlib
 import logging
 import pkgutil
 from enum import Enum
-from typing import Annotated, Union
+from typing import Annotated, Union, get_args
 
 from pydantic import Field
 
@@ -39,3 +39,21 @@ DetectorConfig = Annotated[
     Union[tuple(BaseDetectorConfig.__subclasses__())],  # noqa: UP007
     Field(discriminator="type"),
 ]
+
+
+def _discriminator_value(config_class: type[BaseDetectorConfig]) -> str | None:
+    """Read the Literal value of a detector config class' type field."""
+    field = config_class.model_fields.get("type")
+
+    if field is None:
+        return None
+
+    values = get_args(field.annotation)
+    return values[0] if values else None
+
+
+config_types: dict[str, type[BaseDetectorConfig]] = {
+    key: config_class
+    for config_class in BaseDetectorConfig.__subclasses__()
+    if (key := _discriminator_value(config_class)) is not None
+}

frigate/detectors/device.py

@@ -0,0 +1,113 @@
+"""Parsing of detection hardware device strings."""
+
+import logging
+from dataclasses import dataclass
+
+from pydantic import TypeAdapter, ValidationError
+
+from frigate.detectors.detector_config import BaseDetectorConfig, ModelConfig
+from frigate.detectors.detector_types import DetectorConfig, config_types
+
+logger = logging.getLogger(__name__)
+
+_detector_adapter: TypeAdapter[BaseDetectorConfig] = TypeAdapter(DetectorConfig)
+
+
+@dataclass(frozen=True)
+class DeviceSpec:
+    """A parsed `<detector>` or `<detector>:<device>` string."""
+
+    raw: str
+    detector: str
+    device: str | None
+
+    @property
+    def shareable(self) -> bool:
+        """Whether this device may be listed more than once."""
+        return config_types[self.detector].shareable
+
+
+class DeviceParseError(ValueError):
+    pass
+
+
+def parse_device(raw: str) -> DeviceSpec:
+    """Parse a device string into its detector type and detector specific device.
+
+    Args:
+        raw: The configured device string, for example 'edgetpu:pci:0'
+
+    Returns:
+        The parsed spec
+
+    Raises:
+        DeviceParseError: If the detector type is unknown or the device is not
+            valid for that detector
+    """
+    detector, separator, device = raw.partition(":")
+
+    if detector not in config_types:
+        raise DeviceParseError(
+            f"'{raw}' does not name a known detector. Available detectors are {', '.join(sorted(config_types))}"
+        )
+
+    spec = DeviceSpec(raw=raw, detector=detector, device=device if separator else None)
+
+    # surface a bad device now rather than when the detection process starts
+    build_detector_config(spec, None)
+    return spec
+
+
+def build_detector_config(
+    spec: DeviceSpec, model: ModelConfig | None
+) -> BaseDetectorConfig:
+    """Build the detector config a device string describes.
+
+    Args:
+        spec: The parsed device spec
+        model: The model this detector runs, if it has been resolved yet
+
+    Returns:
+        The validated detector config
+
+    Raises:
+        DeviceParseError: If the device is not valid for this detector type
+    """
+    config: dict[str, object] = {"type": spec.detector, "model": model}
+
+    if spec.device is not None:
+        config_class = config_types[spec.detector]
+
+        try:
+            config[config_class.device_spec_field] = config_class.device_spec_type(
+                spec.device
+            )
+        except ValueError as err:
+            raise DeviceParseError(
+                f"'{spec.raw}' is not a valid {spec.detector} device: {err}"
+            ) from err
+
+    try:
+        return _detector_adapter.validate_python(config)
+    except ValidationError as err:
+        raise DeviceParseError(f"'{spec.raw}' is not a valid device: {err}") from err
+
+
+def runner_names(devices: list[DeviceSpec]) -> list[str]:
+    """Build a unique name for each device, since a shareable device may repeat.
+
+    Args:
+        devices: Every device spec across every configured model, in config order
+
+    Returns:
+        A name per device, suffixed with '#2', '#3', etc. on repeats
+    """
+    names: list[str] = []
+    seen: dict[str, int] = {}
+
+    for spec in devices:
+        count = seen.get(spec.raw, 0) + 1
+        seen[spec.raw] = count
+        names.append(spec.raw if count == 1 else f"{spec.raw}#{count}")
+
+    return names

frigate/detectors/plugins/cpu_tfl.py

@@ -1,5 +1,5 @@
 import logging
-from typing import Literal
+from typing import ClassVar, Literal
 
 from pydantic import ConfigDict, Field
 
@@ -27,6 +27,9 @@ class CpuDetectorConfig(BaseDetectorConfig):
         title="CPU",
     )
 
+    device_spec_field: ClassVar[str] = "num_threads"
+    device_spec_type: ClassVar[type] = int
+
     type: Literal[DETECTOR_KEY]
     num_threads: int = Field(
         default=3,

frigate/detectors/plugins/edgetpu_tfl.py

@@ -1,7 +1,7 @@
 import logging
 import math
 import os
-from typing import Literal
+from typing import ClassVar, Literal
 
 import cv2
 import numpy as np
@@ -28,6 +28,9 @@ class EdgeTpuDetectorConfig(BaseDetectorConfig):
         title="EdgeTPU",
     )
 
+    # a TPU can only be opened by one process
+    shareable: ClassVar[bool] = False
+
     type: Literal[DETECTOR_KEY]
     device: str = Field(
         default=None,

frigate/detectors/plugins/memryx.py

@@ -5,7 +5,7 @@ import shutil
 import urllib.request
 import zipfile
 from queue import Queue
-from typing import Literal
+from typing import ClassVar, Literal
 
 import cv2
 import numpy as np
@@ -37,6 +37,9 @@ class MemryXDetectorConfig(BaseDetectorConfig):
         title="MemryX",
     )
 
+    # an accelerator can only be opened by one process
+    shareable: ClassVar[bool] = False
+
     type: Literal[DETECTOR_KEY]
     device: str = Field(
         default="PCIe",

frigate/detectors/plugins/openvino.py

@@ -28,7 +28,7 @@ class OvDetectorConfig(BaseDetectorConfig):
 
     type: Literal[DETECTOR_KEY]
     device: str = Field(
-        default=None,
+        default="AUTO",
         title="Device Type",
         description="The device to use for OpenVINO inference (e.g. 'CPU', 'GPU', 'NPU').",
     )

frigate/detectors/plugins/rknn.py

@@ -2,7 +2,7 @@ import logging
 import os.path
 import re
 import urllib.request
-from typing import Literal
+from typing import ClassVar, Literal
 
 import cv2
 import numpy as np
@@ -35,6 +35,9 @@ class RknnDetectorConfig(BaseDetectorConfig):
         title="RKNN",
     )
 
+    device_spec_field: ClassVar[str] = "num_cores"
+    device_spec_type: ClassVar[type] = int
+
     type: Literal[DETECTOR_KEY]
     num_cores: int = Field(
         default=0,

frigate/detectors/plugins/tensorrt.py

@@ -14,7 +14,7 @@ try:
 except ModuleNotFoundError:
     TRT_SUPPORT = False
 
-from typing import Literal
+from typing import ClassVar, Literal
 
 from pydantic import ConfigDict, Field
 
@@ -53,6 +53,8 @@ class TensorRTDetectorConfig(BaseDetectorConfig):
         title="TensorRT",
     )
 
+    device_spec_type: ClassVar[type] = int
+
     type: Literal[DETECTOR_KEY]
     device: int = Field(
         default=0, title="GPU Device Index", description="The GPU device index to use."