diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index 874b03892..0c8a8b7ea 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -360,6 +360,11 @@ class CustomCollector(object): "GPU encoder utilisation %", labels=["gpu_name"], ) + gpu_compute_usages = GaugeMetricFamily( + "frigate_gpu_compute_usage_percent", + "GPU compute / encode utilisation %", + labels=["gpu_name"], + ) gpu_dec_usages = GaugeMetricFamily( "frigate_gpu_decoder_usage_percent", "GPU decoder utilisation %", @@ -371,6 +376,9 @@ class CustomCollector(object): self.add_metric(gpu_usages, [gpu_name], gpu_stats, "gpu") self.add_metric(gpu_mem_usages, [gpu_name], gpu_stats, "mem") self.add_metric(gpu_enc_usages, [gpu_name], gpu_stats, "enc") + self.add_metric( + gpu_compute_usages, [gpu_name], gpu_stats, "compute" + ) self.add_metric(gpu_dec_usages, [gpu_name], gpu_stats, "dec") except KeyError: pass @@ -378,6 +386,7 @@ class CustomCollector(object): yield gpu_usages yield gpu_mem_usages yield gpu_enc_usages + yield gpu_compute_usages yield gpu_dec_usages # service stats diff --git a/frigate/test/test_gpu_stats.py b/frigate/test/test_gpu_stats.py index 554077834..2604c4002 100644 --- a/frigate/test/test_gpu_stats.py +++ b/frigate/test/test_gpu_stats.py @@ -45,6 +45,6 @@ class TestGpuStats(unittest.TestCase): assert intel_stats == { "gpu": "26.08%", "mem": "-%", - "enc": "0.0%", + "compute": "0.0%", "dec": "2.27%", } diff --git a/frigate/util/services.py b/frigate/util/services.py index 9b3f9f106..f0bf2de1e 100644 --- a/frigate/util/services.py +++ b/frigate/util/services.py @@ -296,7 +296,7 @@ def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, s render.append(float(single)) if render: - results["enc"] = f"{round(sum(render) / len(render), 2)}%" + results["compute"] = f"{round(sum(render) / len(render), 2)}%" # Video engines are the fixed-function decode engines video = [] @@ -395,9 +395,9 @@ def get_intel_gpu_stats(intel_gpu_device: Optional[str]) -> Optional[dict[str, s results["mem"] = "-%" - # Encoder: Render/3D engine (compute/shader encode) + # Compute: Render/3D engine (compute/shader workloads and QSV encode) if render_global: - results["enc"] = f"{round(sum(render_global) / len(render_global), 2)}%" + results["compute"] = f"{round(sum(render_global) / len(render_global), 2)}%" # Decoder: Video engine (fixed-function codec) if video_global: diff --git a/web/public/locales/en/views/system.json b/web/public/locales/en/views/system.json index 0e3d6a35e..6c3f37f71 100644 --- a/web/public/locales/en/views/system.json +++ b/web/public/locales/en/views/system.json @@ -78,6 +78,7 @@ "gpuUsage": "GPU Usage", "gpuMemory": "GPU Memory", "gpuEncoder": "GPU Encoder", + "gpuCompute": "GPU Compute / Encode", "gpuDecoder": "GPU Decoder", "gpuTemperature": "GPU Temperature", "gpuInfo": { @@ -188,6 +189,7 @@ "cameraFfmpeg": "{{camName}} FFmpeg", "cameraCapture": "{{camName}} capture", "cameraDetect": "{{camName}} detect", + "cameraGpu": "{{camName}} GPU", "cameraFramesPerSecond": "{{camName}} frames per second", "cameraDetectionsPerSecond": "{{camName}} detections per second", "cameraSkippedDetectionsPerSecond": "{{camName}} skipped detections per second" diff --git a/web/src/types/stats.ts b/web/src/types/stats.ts index 1046e0b47..c4b811185 100644 --- a/web/src/types/stats.ts +++ b/web/src/types/stats.ts @@ -60,8 +60,10 @@ export type GpuStats = { mem: string; enc?: string; dec?: string; + compute?: string; pstate?: string; temp?: number; + clients?: { [pid: string]: string }; }; export type NpuStats = { diff --git a/web/src/views/system/CameraMetrics.tsx b/web/src/views/system/CameraMetrics.tsx index b6c5be4fa..e10a437b2 100644 --- a/web/src/views/system/CameraMetrics.tsx +++ b/web/src/views/system/CameraMetrics.tsx @@ -3,6 +3,7 @@ import { CameraLineGraph } from "@/components/graph/LineGraph"; import CameraInfoDialog from "@/components/overlay/CameraInfoDialog"; import { ConnectionQualityIndicator } from "@/components/camera/ConnectionQualityIndicator"; import { Skeleton } from "@/components/ui/skeleton"; +import { cn } from "@/lib/utils"; import { FrigateConfig } from "@/types/frigateConfig"; import { FrigateStats } from "@/types/stats"; import { useCallback, useEffect, useMemo, useState } from "react"; @@ -43,7 +44,7 @@ export default function CameraMetrics({ [ "stats/history", { - keys: "cpu_usages,cameras,camera_fps,detection_fps,skipped_fps,service", + keys: "cpu_usages,gpu_usages,cameras,camera_fps,detection_fps,skipped_fps,service", }, ], { @@ -183,6 +184,64 @@ export default function CameraMetrics({ return series; }, [config, getCameraName, statsHistory, t]); + const cameraGpuSeries = useMemo(() => { + if (!statsHistory || statsHistory.length == 0) { + return {}; + } + + // Find GPU entry with per-client data + const firstGpu = statsHistory.find((s) => s?.gpu_usages); + if (!firstGpu?.gpu_usages) { + return {}; + } + + const gpuWithClients = Object.values(firstGpu.gpu_usages).find( + (g) => g.clients, + ); + if (!gpuWithClients) { + return {}; + } + + const series: { + [cam: string]: { + [key: string]: { name: string; data: { x: number; y: string }[] }; + }; + } = {}; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + const gpuClients = Object.values(stats.gpu_usages || {}).find( + (g) => g.clients, + )?.clients; + + Object.entries(stats.cameras).forEach(([key, camStats]) => { + if (!config?.cameras[key].enabled) { + return; + } + + if (!(key in series)) { + const camName = getCameraName(key); + series[key] = {}; + series[key]["gpu"] = { + name: t("cameras.label.cameraGpu", { camName: camName }), + data: [], + }; + } + + const pid = camStats.ffmpeg_pid?.toString(); + series[key]["gpu"].data.push({ + x: statsIdx, + y: (gpuClients && pid ? gpuClients[pid] : undefined) + ?.slice(0, -1) ?? "0", + }); + }); + }); + return series; + }, [config, getCameraName, statsHistory, t]); + const cameraFpsSeries = useMemo(() => { if (!statsHistory) { return {}; @@ -332,7 +391,11 @@ export default function CameraMetrics({
0 && + "lg:grid-cols-3", + )} > {Object.keys(cameraCpuSeries).includes(camera.name) ? (
@@ -350,6 +413,20 @@ export default function CameraMetrics({ ) : ( )} + {Object.keys(cameraGpuSeries).includes(camera.name) && ( +
+
GPU
+ +
+ )} {Object.keys(cameraFpsSeries).includes(camera.name) ? (
diff --git a/web/src/views/system/GeneralMetrics.tsx b/web/src/views/system/GeneralMetrics.tsx index e78486767..fc7410b5a 100644 --- a/web/src/views/system/GeneralMetrics.tsx +++ b/web/src/views/system/GeneralMetrics.tsx @@ -334,6 +334,43 @@ export default function GeneralMetrics({ return Object.keys(series).length > 0 ? Object.values(series) : undefined; }, [statsHistory]); + const gpuComputeSeries = useMemo(() => { + if (!statsHistory) { + return []; + } + + const series: { + [key: string]: { name: string; data: { x: number; y: string }[] }; + } = {}; + let hasValidGpu = false; + + statsHistory.forEach((stats, statsIdx) => { + if (!stats) { + return; + } + + Object.entries(stats.gpu_usages || {}).forEach(([key, stats]) => { + if (!(key in series)) { + series[key] = { name: key, data: [] }; + } + + if (stats.compute) { + hasValidGpu = true; + series[key].data.push({ + x: statsIdx + 1, + y: stats.compute.slice(0, -1), + }); + } + }); + }); + + if (!hasValidGpu) { + return []; + } + + return Object.keys(series).length > 0 ? Object.values(series) : undefined; + }, [statsHistory]); + const gpuDecSeries = useMemo(() => { if (!statsHistory) { return []; @@ -742,8 +779,9 @@ export default function GeneralMetrics({ className={cn( "mt-4 grid grid-cols-1 gap-2 sm:grid-cols-2", gpuTempSeries?.length && "md:grid-cols-3", - gpuEncSeries?.length && "xl:grid-cols-4", - gpuEncSeries?.length && + (gpuEncSeries?.length || gpuComputeSeries?.length) && + "xl:grid-cols-4", + (gpuEncSeries?.length || gpuComputeSeries?.length) && gpuTempSeries?.length && "3xl:grid-cols-5", )} @@ -856,6 +894,30 @@ export default function GeneralMetrics({ ) : ( )} + {statsHistory.length != 0 ? ( + <> + {gpuComputeSeries && gpuComputeSeries?.length != 0 && ( +
+
+ {t("general.hardwareInfo.gpuCompute")} +
+ {gpuComputeSeries.map((series) => ( + + ))} +
+ )} + + ) : ( + + )} {statsHistory.length != 0 ? ( <> {gpuDecSeries && gpuDecSeries?.length != 0 && (