mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-22 21:41:50 +03:00
Update DEEPX detector to DX-RT 3.5.0 (#24989)
* Update DEEPX detector to DX-RT 3.5.0 * Keep the DEEPX package for offline reinstalls, update docs.
This commit is contained in:
@@ -16,14 +16,20 @@ from pydantic import ConfigDict, Field, field_validator
|
||||
from frigate.detectors.detection_api import DetectionApi
|
||||
from frigate.detectors.detector_config import BaseDetectorConfig, ModelTypeEnum
|
||||
from frigate.util.model import post_process_yolo, post_process_yolox
|
||||
from frigate.util.runtime_deps import Artifact, ArtifactKind, RuntimeManifest
|
||||
from frigate.util.runtime_deps import (
|
||||
ArchiveDest,
|
||||
ArchiveMapping,
|
||||
Artifact,
|
||||
ArtifactKind,
|
||||
RuntimeManifest,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DETECTOR_KEY = "deepx"
|
||||
|
||||
# Installed at first start; DEEPX's PyPI wheels match the libdxrt-bin build.
|
||||
DXRT_VERSION = "3.4.0"
|
||||
# Installed at first start; the wheel ships inside the libdxrt-bin package.
|
||||
DXRT_VERSION = "3.5.0"
|
||||
|
||||
# Where the host dxrtd accepts clients; a container cannot reach its abstract
|
||||
# socket, and a mounted socket file pins a stale inode after a daemon restart.
|
||||
@@ -70,10 +76,13 @@ PPU_ANCHORS_BY_SCALES = {
|
||||
),
|
||||
}
|
||||
|
||||
# .dxnn: "DXNN", a 4-byte LE version, then a JSON index padded to
|
||||
# DXNN_HEADER_SIZE; every offset in the index counts from there.
|
||||
# .dxnn v7/v8: "DXNN", a 4-byte LE version, then a JSON index padded to
|
||||
# DXNN_HEADER_SIZE, offsets relative to it. v9 prefixes the index length.
|
||||
DXNN_MAGIC = b"DXNN"
|
||||
DXNN_HEADER_SIZE = 8192
|
||||
DXNN_PRELUDE_SIZE = 32
|
||||
DXNN_V9_VERSION = 9
|
||||
DXNN_MAX_HEADER_SIZE = 16 * 1024 * 1024
|
||||
# The graph DX-COM keeps for visualisation, an ONNX ModelProto beside the
|
||||
# encrypted model; only its nodes are read, for how the box tensor was built.
|
||||
DXNN_GRAPH_SECTION = "vis_npu_models"
|
||||
@@ -103,32 +112,45 @@ MULTIPART_CLASSES = 80
|
||||
MULTIPART_CHANNELS = 3 * (5 + MULTIPART_CLASSES)
|
||||
|
||||
|
||||
# PyPI URLs carry a per-file digest, so a version bump rewrites them whole.
|
||||
# The wheel links against the runtime libraries the package ships.
|
||||
DEEPX_LIBS = ArchiveMapping(
|
||||
"./usr/local/lib/",
|
||||
ArchiveDest.lib,
|
||||
include=("libdxrt.so*", "libonnxruntime.so*"),
|
||||
)
|
||||
|
||||
# The deb carries a wheel per CPython ABI; `member` picks this image's and
|
||||
# `member_sha256` pins it. Only the wheel and DEEPX_LIBS are kept out of it.
|
||||
DEEPX_MANIFEST = RuntimeManifest(
|
||||
name=DETECTOR_KEY,
|
||||
version=DXRT_VERSION,
|
||||
artifacts=(
|
||||
Artifact(
|
||||
url=(
|
||||
"https://files.pythonhosted.org/packages/de/45/"
|
||||
"1ca593e1c4ed868618658e07adeffc1fb44b92655d1ba48a6c76942616ee/"
|
||||
"dx_engine-3.4.0-cp311-cp311-manylinux_2_27_aarch64.whl"
|
||||
"https://github.com/DEEPX-AI/dx_rt/raw/refs/tags/v3.5.0/"
|
||||
"release/3.5.0/libdxrt-bin_3.5.0_arm64.deb"
|
||||
),
|
||||
sha256="ec4188e0a598a164bc04312482f70159be1a3d888284dcf52746054138101ec7",
|
||||
kind=ArtifactKind.wheel,
|
||||
sha256="f7c942ddfa325f435de3ab9a42b32b322f97dacff7f3468a37e32a6386a03681",
|
||||
kind=ArtifactKind.deb,
|
||||
machines=("aarch64",),
|
||||
mappings=(DEEPX_LIBS,),
|
||||
member="dx_engine-3.5.0-cp311-cp311-*.whl",
|
||||
member_sha256="276e1860af91ab3f937047fe8c4f59c169b4a5ed191f1c1cb606f89045c6967a",
|
||||
),
|
||||
Artifact(
|
||||
url=(
|
||||
"https://files.pythonhosted.org/packages/b0/32/"
|
||||
"d52b33d5b85f7d8565e3e2d153f19db6d506cd5504339f31c6553eae06c6/"
|
||||
"dx_engine-3.4.0-cp311-cp311-manylinux_2_27_x86_64.whl"
|
||||
"https://github.com/DEEPX-AI/dx_rt/raw/refs/tags/v3.5.0/"
|
||||
"release/3.5.0/libdxrt-bin_3.5.0_amd64.deb"
|
||||
),
|
||||
sha256="161fde8428fc8aea95c560d878bc68e8d1c657c18a0eb04840de4bd212f3477f",
|
||||
kind=ArtifactKind.wheel,
|
||||
sha256="c1993416e43059fb4751b2fd093d4232ab2ae8e67a5ec4156c79889e66260f64",
|
||||
kind=ArtifactKind.deb,
|
||||
machines=("x86_64",),
|
||||
mappings=(DEEPX_LIBS,),
|
||||
member="dx_engine-3.5.0-cp311-cp311-*.whl",
|
||||
member_sha256="ec5b80970271ac6d236fa4bc3277e425d6d8bcb89a977f7c236ac0431dd60558",
|
||||
),
|
||||
),
|
||||
preload=("libonnxruntime.so.1", "libdxrt.so"),
|
||||
import_check="dx_engine",
|
||||
)
|
||||
|
||||
@@ -408,17 +430,32 @@ def read_ppu_layout(path: str) -> PpuLayout | None:
|
||||
is none to read."""
|
||||
try:
|
||||
with open(path, "rb") as model:
|
||||
header = model.read(DXNN_HEADER_SIZE)
|
||||
if header[:4] != DXNN_MAGIC:
|
||||
prelude = model.read(DXNN_PRELUDE_SIZE)
|
||||
if prelude[:4] != DXNN_MAGIC:
|
||||
return None
|
||||
|
||||
index = json.JSONDecoder().raw_decode(
|
||||
header[8:].decode("utf-8", "replace")
|
||||
)[0]
|
||||
version = struct.unpack_from("<I", prelude, 4)[0]
|
||||
|
||||
if version >= DXNN_V9_VERSION:
|
||||
header_length = struct.unpack_from("<Q", prelude, 8)[0]
|
||||
|
||||
if header_length > DXNN_MAX_HEADER_SIZE:
|
||||
return None
|
||||
|
||||
raw_index = model.read(header_length)
|
||||
data_offset = DXNN_PRELUDE_SIZE + header_length
|
||||
else:
|
||||
model.seek(8)
|
||||
raw_index = model.read(DXNN_HEADER_SIZE - 8)
|
||||
data_offset = DXNN_HEADER_SIZE
|
||||
|
||||
decoder = json.JSONDecoder()
|
||||
index = decoder.raw_decode(raw_index.decode("utf-8", "replace"))[0]
|
||||
|
||||
data = index["data"]
|
||||
|
||||
def section(entry: dict) -> bytes:
|
||||
model.seek(DXNN_HEADER_SIZE + int(entry["offset"]))
|
||||
model.seek(data_offset + int(entry["offset"]))
|
||||
return model.read(int(entry["size"]))
|
||||
|
||||
anchor_based = None
|
||||
|
||||
+73
-10
@@ -6,6 +6,7 @@ import struct
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
import zlib
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import numpy as np
|
||||
@@ -19,6 +20,8 @@ from frigate.detectors.device import (
|
||||
)
|
||||
from frigate.detectors.plugins.deepx import (
|
||||
DEEPX_MANIFEST,
|
||||
DXNN_MAX_HEADER_SIZE,
|
||||
DXNN_PRELUDE_SIZE,
|
||||
DXRT_VERSION,
|
||||
PPU_RECORD_SIZE,
|
||||
DeepxDetector,
|
||||
@@ -33,6 +36,7 @@ from frigate.detectors.plugins.deepx import (
|
||||
resolve_device,
|
||||
validate_yolox_outputs,
|
||||
)
|
||||
from frigate.util.runtime_deps import ArtifactKind
|
||||
|
||||
|
||||
def model_with_type(model_type) -> ModelConfig:
|
||||
@@ -135,7 +139,13 @@ def onnx_box_graph(box_format: str) -> bytes:
|
||||
|
||||
|
||||
def write_dxnn(
|
||||
directory, ppu, layers, name="model.dxnn", table=True, box_format=None
|
||||
directory,
|
||||
ppu,
|
||||
layers,
|
||||
name="model.dxnn",
|
||||
table=True,
|
||||
box_format=None,
|
||||
version=None,
|
||||
) -> str:
|
||||
"""A minimal .dxnn as DX-RT's parsers read it: the container header, a
|
||||
compile_config carrying `ppu`, and either the PPU tensor table with one
|
||||
@@ -145,7 +155,9 @@ def write_dxnn(
|
||||
anchor-free scale has one entry, and grid_h 1 means a flattened
|
||||
(1, cells, channels) tensor. `box_format`, "centre" or "corner", adds
|
||||
the compiled graph that says how the head writes its boxes, along with
|
||||
the compile_config layer naming the node the PPU reads them from."""
|
||||
the compile_config layer naming the node the PPU reads them from.
|
||||
`version` overrides the format, 9 replacing the padded index with a
|
||||
length-prefixed one."""
|
||||
if box_format is not None and "layer" not in ppu:
|
||||
ppu = dict(ppu, layer=[{"bbox": PPU_BBOX_NODE, "cls_conf": "/head/Sigmoid"}])
|
||||
|
||||
@@ -209,9 +221,12 @@ def write_dxnn(
|
||||
}
|
||||
}
|
||||
|
||||
if version is None:
|
||||
version = 8 if table else 7
|
||||
|
||||
index = json.dumps(
|
||||
{
|
||||
"version": 8 if table else 7,
|
||||
"version": version,
|
||||
"signature": "DXNN",
|
||||
"size": 8192,
|
||||
"data": data,
|
||||
@@ -220,8 +235,17 @@ def write_dxnn(
|
||||
|
||||
path = os.path.join(directory, name)
|
||||
with open(path, "wb") as model:
|
||||
model.write(b"DXNN" + struct.pack("<I", 8))
|
||||
model.write(index.ljust(8192 - 8, b"\0"))
|
||||
if version >= 9:
|
||||
model.write(b"DXNN")
|
||||
model.write(struct.pack("<I", version))
|
||||
model.write(struct.pack("<Q", len(index)))
|
||||
model.write(struct.pack("<I", zlib.crc32(index)))
|
||||
model.write(b"\0" * 12) # flags and reserved bytes
|
||||
model.write(index)
|
||||
else:
|
||||
model.write(b"DXNN" + struct.pack("<I", version))
|
||||
model.write(index.ljust(8192 - 8, b"\0"))
|
||||
|
||||
model.write(compile_config + part + graph)
|
||||
|
||||
return path
|
||||
@@ -324,6 +348,40 @@ class TestDeepxModelFile(unittest.TestCase):
|
||||
with self.subTest(graph=graph):
|
||||
self.assertIsNone(self.layout(ppu, layers, **kwargs).centre_boxes)
|
||||
|
||||
def test_a_v9_file_is_read_from_its_variable_length_index(self):
|
||||
cases = {
|
||||
"table": (ANCHOR_BASED_PPU, THREE_SCALE_ANCHORS, {"table": True}),
|
||||
"no table: compile_config sits past the index": (
|
||||
{"num_classes": 80},
|
||||
THREE_SCALE_ANCHORS,
|
||||
{"table": False},
|
||||
),
|
||||
}
|
||||
|
||||
for kind, (ppu, layers, kwargs) in cases.items():
|
||||
with self.subTest(kind=kind):
|
||||
self.assertEqual(
|
||||
self.layout(ppu, layers, version=9, **kwargs),
|
||||
PpuLayout(anchor_based=True, grids=((80, 80), (40, 40), (20, 20))),
|
||||
)
|
||||
|
||||
def test_a_v9_index_shorter_than_it_claims_is_refused(self):
|
||||
path = write_dxnn(self.tmp.name, ANCHOR_BASED_PPU, [(80, 80, 3)], version=9)
|
||||
|
||||
with open(path, "r+b") as model:
|
||||
model.truncate(DXNN_PRELUDE_SIZE + 8)
|
||||
|
||||
self.assertIsNone(read_ppu_layout(path))
|
||||
|
||||
def test_a_v9_index_larger_than_the_cap_is_refused(self):
|
||||
path = write_dxnn(self.tmp.name, ANCHOR_BASED_PPU, [(80, 80, 3)], version=9)
|
||||
|
||||
with open(path, "r+b") as model:
|
||||
model.seek(8)
|
||||
model.write(struct.pack("<Q", DXNN_MAX_HEADER_SIZE + 1))
|
||||
|
||||
self.assertIsNone(read_ppu_layout(path))
|
||||
|
||||
def test_nothing_is_read_from_a_model_without_ppu_metadata(self):
|
||||
self.assertIsNone(read_ppu_layout(os.path.join(self.tmp.name, "missing")))
|
||||
|
||||
@@ -607,15 +665,20 @@ class TestDeepxConfig(unittest.TestCase):
|
||||
self.assertEqual(config.device, "PCIe:0")
|
||||
self.assertEqual(config.model.model_type, ModelTypeEnum.yologeneric)
|
||||
|
||||
def test_the_runtime_manifest_pins_its_wheels_to_the_version(self):
|
||||
"""A PyPI path carries a per-file digest, so bumping DXRT_VERSION has
|
||||
to rewrite the whole URL; a stale one installs the old wheel and fails
|
||||
the sha256 on every user's first start."""
|
||||
def test_the_runtime_manifest_pins_the_package_and_wheel_to_the_version(self):
|
||||
"""The deb carries the wheel, so bumping DXRT_VERSION has to rewrite
|
||||
the package URL, both digests and the wheel member together."""
|
||||
self.assertEqual(DEEPX_MANIFEST.version, DXRT_VERSION)
|
||||
|
||||
for artifact in DEEPX_MANIFEST.artifacts:
|
||||
with self.subTest(url=artifact.url):
|
||||
self.assertIn(f"dx_engine-{DXRT_VERSION}-", artifact.url)
|
||||
self.assertIs(artifact.kind, ArtifactKind.deb)
|
||||
self.assertIn(f"libdxrt-bin_{DXRT_VERSION}_", artifact.url)
|
||||
self.assertIn(f"dx_engine-{DXRT_VERSION}-cp311-", artifact.member)
|
||||
self.assertEqual(len(artifact.member_sha256), 64)
|
||||
self.assertTrue(artifact.mappings)
|
||||
|
||||
self.assertIn("libdxrt.so", DEEPX_MANIFEST.preload)
|
||||
|
||||
def test_a_model_less_config_still_validates(self):
|
||||
config = DeepxDetectorConfig(type="deepx")
|
||||
|
||||
@@ -26,6 +26,8 @@ import sys
|
||||
import tarfile
|
||||
import tempfile
|
||||
import zipfile
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from dataclasses import dataclass
|
||||
from enum import StrEnum
|
||||
from pathlib import Path
|
||||
@@ -50,6 +52,7 @@ class RuntimeDependencyError(RuntimeError):
|
||||
class ArtifactKind(StrEnum):
|
||||
wheel = "wheel"
|
||||
archive = "archive"
|
||||
deb = "deb" # a Debian package that carries a wheel
|
||||
|
||||
|
||||
class ArchiveDest(StrEnum):
|
||||
@@ -82,6 +85,10 @@ class Artifact:
|
||||
filename: str | None = None
|
||||
mappings: tuple[ArchiveMapping, ...] = ()
|
||||
machines: tuple[str, ...] = ()
|
||||
# basename glob for the wheel a `deb` carries
|
||||
member: str = ""
|
||||
# sha256 of that wheel, pinning the pip input like a wheel artifact does
|
||||
member_sha256: str = ""
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
@@ -228,6 +235,106 @@ def _stage(path: Path, artifact: Artifact, staging: Path) -> Path:
|
||||
return staged
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _deb_contents(path: Path) -> Iterator[tarfile.TarFile]:
|
||||
"""Stream the filesystem tarball inside a Debian package.
|
||||
|
||||
dpkg-deb decompresses data.tar to stdout, so no archive hits the disk.
|
||||
"""
|
||||
stderr = tempfile.TemporaryFile()
|
||||
|
||||
try:
|
||||
process = subprocess.Popen(
|
||||
["dpkg-deb", "--fsys-tarfile", str(path)],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=stderr,
|
||||
)
|
||||
except FileNotFoundError as err:
|
||||
stderr.close()
|
||||
raise RuntimeDependencyError(
|
||||
f"dpkg-deb is required to unpack {path.name}; the Frigate image "
|
||||
"ships it as part of dpkg"
|
||||
) from err
|
||||
|
||||
try:
|
||||
assert process.stdout is not None
|
||||
with tarfile.open(fileobj=process.stdout, mode="r|") as archive:
|
||||
yield archive
|
||||
finally:
|
||||
if process.stdout is not None:
|
||||
process.stdout.close()
|
||||
|
||||
returncode = process.wait()
|
||||
stderr.seek(0)
|
||||
message = stderr.read().decode("utf-8", "replace").strip()
|
||||
stderr.close()
|
||||
|
||||
if returncode != 0:
|
||||
raise RuntimeDependencyError(
|
||||
f"Unable to unpack {path.name}: {message[-2000:] or 'dpkg-deb failed'}"
|
||||
)
|
||||
|
||||
|
||||
def _install_deb(path: Path, artifact: Artifact, staging: Path) -> list[str]:
|
||||
"""Install a Debian package's mapped files and the wheel it carries.
|
||||
|
||||
The rest of the package belongs on the host, not in the image. The
|
||||
package itself stays in the cache so later reinstalls need no download.
|
||||
"""
|
||||
machine = platform.machine()
|
||||
mappings = [m for m in artifact.mappings if not m.machines or machine in m.machines]
|
||||
written: list[str] = []
|
||||
wheel: Path | None = None
|
||||
|
||||
with _deb_contents(path) as archive:
|
||||
for member in archive:
|
||||
if not (member.isfile() or member.issym()):
|
||||
continue
|
||||
|
||||
for mapping, relative in _mappings_for(member.name, mappings):
|
||||
dest = _safe_dest(_dest_root(mapping), relative)
|
||||
|
||||
if member.issym():
|
||||
_write_symlink(dest, member.linkname)
|
||||
else:
|
||||
source = archive.extractfile(member)
|
||||
assert source is not None
|
||||
with source:
|
||||
_write_member(dest, source, member.mode)
|
||||
|
||||
written.append(str(dest))
|
||||
|
||||
if not artifact.member or wheel is not None or not member.isfile():
|
||||
continue
|
||||
|
||||
name = os.path.basename(member.name)
|
||||
|
||||
if not fnmatch.fnmatch(name, artifact.member):
|
||||
continue
|
||||
|
||||
source = archive.extractfile(member)
|
||||
assert source is not None
|
||||
wheel = staging / name
|
||||
|
||||
with source, open(wheel, "wb") as f:
|
||||
shutil.copyfileobj(source, f)
|
||||
|
||||
if artifact.member and wheel is None:
|
||||
raise RuntimeDependencyError(
|
||||
f"{path.name} does not contain a file matching {artifact.member}"
|
||||
)
|
||||
|
||||
if wheel is not None:
|
||||
if artifact.member_sha256 and sha256_of(wheel) != artifact.member_sha256:
|
||||
raise RuntimeDependencyError(
|
||||
f"Checksum mismatch for the wheel carried by {path.name}"
|
||||
)
|
||||
|
||||
_install_wheel(wheel)
|
||||
|
||||
return written
|
||||
|
||||
|
||||
def _install_wheel(path: Path) -> None:
|
||||
result = subprocess.run(
|
||||
[
|
||||
@@ -518,13 +625,16 @@ def ensure_installed(manifest: RuntimeManifest) -> None:
|
||||
_remove_stamped_files(stamp)
|
||||
staging = Path(tempfile.mkdtemp(prefix=f"frigate-{manifest.name}-"))
|
||||
files: list[str] = []
|
||||
artifacts = _artifacts_for_machine(manifest)
|
||||
|
||||
try:
|
||||
for artifact in _artifacts_for_machine(manifest):
|
||||
for artifact in artifacts:
|
||||
staged = _stage(_fetch(artifact, cache), artifact, staging)
|
||||
|
||||
if artifact.kind is ArtifactKind.wheel:
|
||||
_install_wheel(staged)
|
||||
elif artifact.kind is ArtifactKind.deb:
|
||||
files.extend(_install_deb(staged, artifact, staging))
|
||||
else:
|
||||
files.extend(_extract_archive(staged, artifact))
|
||||
except (OSError, tarfile.TarError, zipfile.BadZipFile) as err:
|
||||
|
||||
Reference in New Issue
Block a user