Compare commits

...
2 Commits
Author SHA1 Message Date
Josh Hawkins 1a7078a7a2 fix crash in multi-camera export dialog when a camera group has an invalid or default icon 2026-07-15 11:42:59 -05:00
Josh Hawkins 9d113c0644 improve intel gpu stats error handling and remove stale zero-usage warning
Verify an i915/xe device exists in /sys/class/drm before scanning /proc, and log a distinct warning for each durable failure: an intel_gpu_device that does not resolve, one that resolves to a non-Intel card, an unreadable /proc, and a kernel that publishes no per-client engine counters. A present GPU with no attached DRM clients now reports an idle 0% reading instead of latching the hour-long hwaccel error cooldown, which blanked stats after momentary gaps such as camera restarts. Remove the System page warning that blamed intel_gpu_top for all-zero readings, since 0.18 no longer uses intel_gpu_top and 0% is now a legitimate idle reading.
2026-07-15 11:34:05 -05:00
6 changed files with 324 additions and 119 deletions
+147 -3
View File
@@ -21,8 +21,12 @@ class TestGpuStats(unittest.TestCase):
@patch("frigate.util.services.time.sleep")
@patch("frigate.util.services.time.monotonic")
@patch("frigate.util.services._read_intel_drm_fdinfo")
def test_intel_gpu_stats_fdinfo(self, read_fdinfo, monotonic, sleep, get_names):
@patch("frigate.util.services._enumerate_drm_devices")
def test_intel_gpu_stats_fdinfo(
self, drm_devices, read_fdinfo, monotonic, sleep, get_names
):
# 1 second of wall clock between snapshots
drm_devices.return_value = {"0000:00:02.0": "i915"}
monotonic.side_effect = [0.0, 1.0]
get_names.return_value = {"0000:00:02.0": "Intel Graphics"}
@@ -96,13 +100,15 @@ class TestGpuStats(unittest.TestCase):
@patch("frigate.util.services.time.sleep")
@patch("frigate.util.services.time.monotonic")
@patch("frigate.util.services._read_intel_drm_fdinfo")
@patch("frigate.util.services._enumerate_drm_devices")
def test_intel_gpu_stats_xe_capacity(
self, read_fdinfo, monotonic, sleep, get_names
self, drm_devices, read_fdinfo, monotonic, sleep, get_names
):
# Xe engines report cumulative cycles paired with total cycles, plus a
# per-class capacity. drm-cycles-* is summed across every instance of a
# class, so on Battlemage (capacity 2 for vcs/vecs) busy/total must be
# divided by capacity to land in 0-100%.
drm_devices.return_value = {"0000:03:00.0": "xe"}
monotonic.side_effect = [0.0, 1.0]
get_names.return_value = {"0000:03:00.0": "Intel Arc"}
@@ -150,7 +156,145 @@ class TestGpuStats(unittest.TestCase):
},
}
@patch("frigate.stats.intel_gpu_info.intel_gpu_name_resolver.get_names")
@patch("frigate.util.services.time.sleep")
@patch("frigate.util.services._read_intel_drm_fdinfo")
def test_intel_gpu_stats_no_clients(self, read_fdinfo):
@patch("frigate.util.services._enumerate_drm_devices")
def test_intel_gpu_stats_no_clients_reports_idle(
self, drm_devices, read_fdinfo, sleep, get_names
):
# The device exists but nothing holds it open, e.g. while camera
# processes are restarting. This is an idle state, not an error:
# returning None here would latch the hwaccel error cooldown and
# blank GPU stats for an hour over a momentary gap.
drm_devices.return_value = {"0000:00:02.0": "i915"}
read_fdinfo.return_value = {}
get_names.return_value = {"0000:00:02.0": "Intel Graphics"}
assert get_intel_gpu_stats(None) == {
"0000:00:02.0": {
"name": "Intel Graphics",
"vendor": "intel",
"gpu": "0.0%",
"mem": "-%",
"compute": "0.0%",
"dec": "0.0%",
},
}
# Idle short-circuits before spending the sample window
sleep.assert_not_called()
read_fdinfo.assert_called_once()
@patch("frigate.util.services.time.sleep")
@patch("frigate.util.services._read_intel_drm_fdinfo")
@patch("frigate.util.services._enumerate_drm_devices")
def test_intel_gpu_stats_clients_without_engine_counters(
self, drm_devices, read_fdinfo, sleep
):
# i915 publishes drm-driver/drm-pdev/drm-client-id but no drm-engine-*
# lines while GuC submission is active on kernels older than 6.5, so
# clients are found with nothing to sample. Reporting idle here would
# be a lie, and sampling a second time cannot help.
drm_devices.return_value = {"0000:00:02.0": "i915"}
read_fdinfo.return_value = {
("0000:00:02.0", "48", "1109"): {
"driver": "i915",
"pid": "1109",
"engines": {},
},
("0000:00:02.0", "51", "1258"): {
"driver": "i915",
"pid": "1258",
"engines": {},
},
}
assert get_intel_gpu_stats(None) is None
sleep.assert_not_called()
read_fdinfo.assert_called_once()
@patch("frigate.util.services._read_intel_drm_fdinfo")
@patch("frigate.util.services._enumerate_drm_devices")
def test_intel_gpu_stats_no_intel_device(self, drm_devices, read_fdinfo):
# Only a non-Intel GPU is visible in sysfs; /proc is never scanned
drm_devices.return_value = {"0000:01:00.0": "nvidia"}
assert get_intel_gpu_stats(None) is None
read_fdinfo.assert_not_called()
@patch("frigate.util.services._read_intel_drm_fdinfo")
@patch("frigate.util.services._enumerate_drm_devices")
@patch("frigate.util.services._resolve_intel_gpu_pdev")
def test_intel_gpu_stats_unresolvable_device_hint(
self, resolve_pdev, drm_devices, read_fdinfo
):
# A configured intel_gpu_device that cannot be resolved is a config
# error, not a reason to silently fall back to reporting all GPUs
resolve_pdev.return_value = None
assert get_intel_gpu_stats("/dev/dri/renderD999") is None
drm_devices.assert_not_called()
read_fdinfo.assert_not_called()
@patch("frigate.util.services._read_intel_drm_fdinfo")
@patch("frigate.util.services._enumerate_drm_devices")
@patch("frigate.util.services._resolve_intel_gpu_pdev")
def test_intel_gpu_stats_hint_resolves_to_non_intel_gpu(
self, resolve_pdev, drm_devices, read_fdinfo
):
# card numbering can reorder across reboots on multi-GPU hosts, so a
# configured hint may point at another vendor's card; call it out
# instead of reporting nothing
resolve_pdev.return_value = "0000:01:00.0"
drm_devices.return_value = {
"0000:00:02.0": "i915",
"0000:01:00.0": "nvidia",
}
assert get_intel_gpu_stats("/dev/dri/card0") is None
read_fdinfo.assert_not_called()
@patch("frigate.util.services._read_intel_drm_fdinfo")
@patch("frigate.util.services._enumerate_drm_devices")
def test_intel_gpu_stats_unreadable_proc(self, drm_devices, read_fdinfo):
# A scan failure (None) is a different condition than a scan that
# finds no clients ({}) and must not report idle
drm_devices.return_value = {"0000:00:02.0": "i915"}
read_fdinfo.return_value = None
assert get_intel_gpu_stats(None) is None
@patch("frigate.stats.intel_gpu_info.intel_gpu_name_resolver.get_names")
@patch("frigate.util.services.time.sleep")
@patch("frigate.util.services.time.monotonic")
@patch("frigate.util.services._read_intel_drm_fdinfo")
@patch("frigate.util.services._enumerate_drm_devices")
def test_intel_gpu_stats_clients_lost_between_samples(
self, drm_devices, read_fdinfo, monotonic, sleep, get_names
):
# Clients disappearing during the sample window is transient process
# churn, so report idle rather than latching an error
drm_devices.return_value = {"0000:00:02.0": "i915"}
monotonic.side_effect = [0.0, 1.0]
get_names.return_value = {"0000:00:02.0": "Intel Graphics"}
read_fdinfo.side_effect = [
{
("0000:00:02.0", "1", "100"): {
"driver": "i915",
"pid": "100",
"engines": {"video": (5_000_000_000, 0, 1)},
},
},
{},
]
assert get_intel_gpu_stats(None) == {
"0000:00:02.0": {
"name": "Intel Graphics",
"vendor": "intel",
"gpu": "0.0%",
"mem": "-%",
"compute": "0.0%",
"dec": "0.0%",
},
}
+164 -25
View File
@@ -285,6 +285,10 @@ _XE_ENGINE_KEYS = {
"vecs": "video-enhance",
"ccs": "compute",
}
_INTEL_DRM_DRIVERS = ("i915", "xe")
_PCI_ADDRESS_RE = re.compile(
r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$"
)
def _resolve_intel_gpu_pdev(device: str | None) -> str | None:
@@ -294,29 +298,70 @@ def _resolve_intel_gpu_pdev(device: str | None) -> str | None:
if not device:
return None
if re.match(r"^[0-9a-fA-F]{4}:[0-9a-fA-F]{2}:[0-9a-fA-F]{2}\.[0-9a-fA-F]$", device):
if _PCI_ADDRESS_RE.match(device):
return device
name = os.path.basename(device.rstrip("/"))
try:
return os.path.basename(os.path.realpath(f"/sys/class/drm/{name}/device"))
pdev = os.path.basename(os.path.realpath(f"/sys/class/drm/{name}/device"))
except OSError:
return None
# realpath does not raise on a nonexistent node; it returns the input
# path unchanged, so validate the result actually looks like a PCI
# address before trusting it.
return pdev if _PCI_ADDRESS_RE.match(pdev) else None
def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict:
def _enumerate_drm_devices() -> dict[str, str]:
"""Map each PCI-attached DRM device to its bound kernel driver.
Reads /sys/class/drm, which reflects every GPU on the host even when only
some render nodes are mapped into the container, so device presence can be
verified without /dev access. Returns {pdev: driver}, e.g.
{"0000:00:02.0": "i915"}.
"""
devices: dict[str, str] = {}
try:
entries = os.listdir("/sys/class/drm")
except OSError:
return devices
for entry in entries:
device_dir = f"/sys/class/drm/{entry}/device"
pdev = os.path.basename(os.path.realpath(device_dir))
if not _PCI_ADDRESS_RE.match(pdev):
continue
try:
driver = os.path.basename(os.readlink(f"{device_dir}/driver"))
except OSError:
continue
devices[pdev] = driver
return devices
def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict | None:
"""Snapshot DRM fdinfo for every Intel client visible in /proc.
Returns a dict keyed by (pdev, drm-client-id, pid) so the same context
seen via multiple file descriptors on a single process collapses to one
entry.
entry. Clients whose fdinfo carries no engine counters are still included
with an empty "engines" dict so the caller can distinguish "clients exist
but the kernel publishes no busyness" from "no clients at all". Returns
None when /proc itself cannot be scanned, which is a different failure
than a scan that finds nothing.
"""
snapshot: dict = {}
try:
proc_entries = os.listdir("/proc")
except OSError:
return snapshot
return None
for entry in proc_entries:
if not entry.isdigit():
@@ -400,42 +445,124 @@ def _read_intel_drm_fdinfo(target_pdev: str | None) -> dict:
except (ValueError, IndexError):
continue
if not engines:
continue
snapshot[key] = {"driver": driver, "pid": entry, "engines": engines}
return snapshot
def _idle_intel_gpu_stats(
target_pdev: str | None, intel_pdevs: dict[str, str]
) -> dict[str, dict[str, Any]]:
"""Build a 0% reading for the configured (or every) Intel GPU.
Used when the device is confirmed present but no DRM client is currently
attached, e.g. while camera processes are restarting. That is an idle
state, not a collection failure, so it must produce a valid reading:
returning None would latch the hwaccel error cooldown and blank GPU stats
for an hour over a momentary gap.
"""
from frigate.stats.intel_gpu_info import intel_gpu_name_resolver
names = intel_gpu_name_resolver.get_names()
pdevs = [target_pdev] if target_pdev else sorted(intel_pdevs)
return {
pdev: {
"name": names.get(pdev) or "Intel iGPU",
"vendor": "intel",
"gpu": "0.0%",
"mem": "-%",
"compute": "0.0%",
"dec": "0.0%",
}
for pdev in pdevs
}
def get_intel_gpu_stats(
intel_gpu_device: str | None,
) -> dict[str, dict[str, Any]] | None:
"""Get stats by reading DRM fdinfo files, bucketed per-pdev.
Each DRM client FD exposes monotonic per-engine busy counters via
/proc/<pid>/fdinfo/<fd> (i915 since kernel 5.19, Xe since first release).
We sample twice and divide busy-time deltas by wall-clock to derive
utilization. Render/3D and Compute are pooled into "compute"; Video and
VideoEnhance into "dec". Overall "gpu" is the sum of those pools (clamped
to 100%).
/proc/<pid>/fdinfo/<fd>. For i915 this requires kernel 6.5 or newer:
earlier kernels omit the per-engine counters whenever GuC submission is
active, which is the default on 12th gen and newer. Xe has exposed them
since its first release. We sample twice and divide busy-time deltas by
wall-clock to derive utilization. Render/3D and Compute are pooled into
"compute"; Video and VideoEnhance into "dec". Overall "gpu" is the sum of
those pools (clamped to 100%).
The return value is keyed by the GPU's drm-pdev string so multiple Intel
GPUs in the same system are reported separately. Each entry carries a
"name" populated from OpenVINO (falling back to the pdev) so callers can
surface a real device name in the UI.
A device that exists but has no attached DRM clients reports an idle 0%
reading. None is returned only for durable failures (no Intel GPU, a bad
intel_gpu_device config, unreadable /proc, or a kernel that publishes no
counters), each of which logs a distinct warning, and the caller latches
it against retries for an hour.
"""
from frigate.stats.intel_gpu_info import intel_gpu_name_resolver
target_pdev = _resolve_intel_gpu_pdev(intel_gpu_device)
if intel_gpu_device and not target_pdev:
logger.warning(
"Unable to collect Intel GPU stats: configured intel_gpu_device %s "
"does not exist or could not be resolved to a PCI device",
intel_gpu_device,
)
return None
drm_devices = _enumerate_drm_devices()
intel_pdevs = {
pdev: driver
for pdev, driver in drm_devices.items()
if driver in _INTEL_DRM_DRIVERS
}
if not intel_pdevs:
logger.warning(
"Unable to collect Intel GPU stats: no Intel GPU (i915/xe) found in "
"/sys/class/drm. Check that the driver is loaded on the host"
)
return None
if target_pdev and target_pdev not in intel_pdevs:
logger.warning(
"Unable to collect Intel GPU stats: configured intel_gpu_device %s "
"resolved to %s (driver: %s), which is not an Intel GPU",
intel_gpu_device,
target_pdev,
drm_devices.get(target_pdev, "unknown"),
)
return None
snapshot_a = _read_intel_drm_fdinfo(target_pdev)
if snapshot_a is None:
logger.warning("Unable to collect Intel GPU stats: /proc could not be read")
return None
if not snapshot_a:
# No process currently holds the GPU open, e.g. while camera processes
# are restarting. The device is confirmed present, so report idle
# rather than an error; the next stats cycle re-samples normally.
logger.debug("No active DRM clients for Intel GPU, reporting idle")
return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
if not any(client["engines"] for client in snapshot_a.values()):
# Clients exist but the kernel published no busyness for them, so
# there is nothing to sample and a second snapshot would not help.
# i915 suppresses per-client engine counters while GuC submission is
# active on kernels older than 6.5 (kernel commit 1324680a80eb lifted
# this), which covers stock Debian 12 and Ubuntu 22.04 on 12th gen
# and newer.
logger.warning(
"Unable to collect Intel GPU stats: no DRM fdinfo entries found"
"%s. Check kernel version (required >= 6.5) and that /proc is readable"
" and the i915/xe driver is loaded",
f" for pdev {target_pdev}" if target_pdev else "",
"Unable to collect Intel GPU stats: found %d DRM client(s) for %s but "
"no per-engine counters. Kernel 6.5 or newer is required.",
len(snapshot_a),
"/".join(sorted({client["driver"] for client in snapshot_a.values()})),
)
return None
@@ -444,12 +571,18 @@ def get_intel_gpu_stats(
elapsed_ns = (time.monotonic() - start) * 1e9
snapshot_b = _read_intel_drm_fdinfo(target_pdev)
if not snapshot_b or elapsed_ns <= 0:
logger.warning(
"Unable to collect Intel GPU stats: second DRM fdinfo sample was empty"
)
if snapshot_b is None:
logger.warning("Unable to collect Intel GPU stats: /proc could not be read")
return None
if not snapshot_b or elapsed_ns <= 0:
# Every client disappeared during the sample window; transient by
# definition, so report idle instead of latching an error.
logger.debug(
"No DRM clients persisted across Intel GPU samples, reporting idle"
)
return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
def _new_engine_pct() -> dict[str, float]:
return {"render": 0.0, "video": 0.0, "video-enhance": 0.0, "compute": 0.0}
@@ -461,6 +594,11 @@ def get_intel_gpu_stats(
if not data_a or data_a["driver"] != data_b["driver"]:
continue
# Skip before the setdefault below so a counter-less client cannot
# register its pdev on its own.
if not data_b["engines"]:
continue
pdev = key[0]
engine_pct = per_pdev_engine_pct.setdefault(pdev, _new_engine_pct())
pid_pct = per_pdev_pid_pct.setdefault(pdev, {})
@@ -492,11 +630,12 @@ def get_intel_gpu_stats(
pid_pct[data_b["pid"]] = pid_pct.get(data_b["pid"], 0.0) + client_total
if not per_pdev_engine_pct:
logger.warning(
"Unable to collect Intel GPU stats: no per-engine counters available "
"(i915 requires kernel >= 5.19)"
# Clients were seen in both snapshots but none persisted as the same
# (pdev, client-id, pid); process churn, so report idle.
logger.debug(
"No DRM clients persisted across Intel GPU samples, reporting idle"
)
return None
return _idle_intel_gpu_stats(target_pdev, intel_pdevs)
names = intel_gpu_name_resolver.get_names()
results: dict[str, dict[str, Any]] = {}
+1 -6
View File
@@ -107,12 +107,7 @@
},
"npuUsage": "NPU Usage",
"npuMemory": "NPU Memory",
"npuTemperature": "NPU Temperature",
"intelGpuWarning": {
"title": "Intel GPU Stats Warning",
"message": "GPU stats unavailable",
"description": "This is a known bug in Intel's GPU stats reporting tools (intel_gpu_top) where it will break and repeatedly return a GPU usage of 0% even in cases where hardware acceleration and object detection are correctly running on the (i)GPU. This is not a Frigate bug. You can restart the host to temporarily fix the issue and confirm that the GPU is working correctly. This does not affect performance."
}
"npuTemperature": "NPU Temperature"
},
"otherProcesses": {
"title": "Other Processes",
+5 -1
View File
@@ -153,11 +153,15 @@ export default function IconPicker({
}
type IconRendererProps = {
icon: IconType;
icon: IconType | undefined;
size?: number;
className?: string;
};
export function IconRenderer({ icon, size, className }: IconRendererProps) {
if (!icon) {
return null;
}
return <>{React.createElement(icon, { size, className })}</>;
}
+6 -1
View File
@@ -65,6 +65,7 @@ import { Textarea } from "../ui/textarea";
import { useNavigate } from "react-router-dom";
import { useIsAdmin } from "@/hooks/use-is-admin";
import { isReplayCamera } from "@/utils/cameraUtil";
import { isValidIconName } from "@/utils/iconUtil";
const EXPORT_OPTIONS = [
"1",
@@ -1066,7 +1067,11 @@ export function ExportContent({
}
>
<IconRenderer
icon={LuIcons[group.icon]}
icon={
isValidIconName(group.icon)
? LuIcons[group.icon]
: LuIcons.LuFolder
}
className="mr-2 size-4 text-secondary-foreground"
/>
<span className="truncate">{group.name}</span>
+1 -83
View File
@@ -470,50 +470,6 @@ export default function GeneralMetrics({
return Object.keys(series).length > 0 ? Object.values(series) : undefined;
}, [statsHistory]);
// Check if Intel GPU has all 0% usage values (known bug)
const showIntelGpuWarning = useMemo(() => {
if (!statsHistory || statsHistory.length < 3) {
return false;
}
const hasIntelGpu = Object.values(statsHistory[0]?.gpu_usages ?? {}).some(
(stats) => stats.vendor === "intel",
);
if (!hasIntelGpu) {
return false;
}
// Check if all GPU usage values are 0% across all stats
let allZero = true;
let hasDataPoints = false;
for (const stats of statsHistory) {
if (!stats) {
continue;
}
Object.values(stats.gpu_usages || {}).forEach((gpuStats) => {
if (gpuStats.vendor !== "intel") {
return;
}
if (gpuStats.gpu) {
hasDataPoints = true;
const gpuValue = parseFloat(gpuStats.gpu.slice(0, -1));
if (!isNaN(gpuValue) && gpuValue > 0) {
allZero = false;
}
}
});
if (!allZero) {
break;
}
}
return hasDataPoints && allZero;
}, [statsHistory]);
// npu stats
const npuSeries = useMemo(() => {
@@ -819,46 +775,8 @@ export default function GeneralMetrics({
<>
{statsHistory.length != 0 ? (
<div className="rounded-lg bg-background_alt p-2.5 md:rounded-2xl">
<div className="mb-5 flex flex-row items-center justify-between">
<div className="mb-5">
{t("general.hardwareInfo.gpuUsage")}
{showIntelGpuWarning && (
<Popover>
<PopoverTrigger asChild>
<button
className="flex flex-row items-center gap-1.5 text-yellow-600 focus:outline-none dark:text-yellow-500"
aria-label={t(
"general.hardwareInfo.intelGpuWarning.title",
)}
>
<CiCircleAlert
className="size-5"
aria-label={t(
"general.hardwareInfo.intelGpuWarning.title",
)}
/>
<span className="text-sm">
{t(
"general.hardwareInfo.intelGpuWarning.message",
)}
</span>
</button>
</PopoverTrigger>
<PopoverContent className="w-80">
<div className="space-y-2">
<div className="font-semibold">
{t(
"general.hardwareInfo.intelGpuWarning.title",
)}
</div>
<div>
{t(
"general.hardwareInfo.intelGpuWarning.description",
)}
</div>
</div>
</PopoverContent>
</Popover>
)}
</div>
{gpuSeries.map((series) => (
<ThresholdBarGraph