Improve System Health pane (#24188)

* build out system health pane

* tweaks

* fixes

* fix notice link so it opens the correct camera

* tweak language

web/src/hooks/use-hardware-health.ts

@@ -0,0 +1,67 @@
+import { useMemo } from "react";
+import useSWR from "swr";
+import { useTranslation } from "react-i18next";
+import type {
+  DetectionHardware,
+  HwaccelRecommendation,
+} from "@/types/hardware";
+import type { FrigateConfig } from "@/types/frigateConfig";
+import { useAutoFrigateStats } from "@/hooks/use-stats";
+import {
+  cameraConnectionCells,
+  detectionRows,
+  enrichmentRows,
+  hwaccelRows,
+  isStartupWindow,
+} from "@/utils/health";
+
+export function useHardwareHealth() {
+  const { t } = useTranslation(["views/system", "views/setup"]);
+  const { data: config } = useSWR<FrigateConfig>("config", {
+    revalidateOnFocus: false,
+  });
+  const { data: hardware, error: probeError } = useSWR<DetectionHardware[]>(
+    "hardware/probe",
+    { revalidateOnFocus: false },
+  );
+  const { data: hwaccel, error: hwaccelError } = useSWR<HwaccelRecommendation>(
+    "hardware/hwaccel",
+    { revalidateOnFocus: false },
+  );
+  const stats = useAutoFrigateStats();
+  const rows = useMemo(() => {
+    if (!config) {
+      return undefined;
+    }
+
+    const startup = isStartupWindow(stats);
+    return {
+      detection: detectionRows({
+        models: config.models,
+        hardware,
+        probeFailed: !!probeError,
+        stats,
+        startup,
+        t,
+      }),
+      hwaccel: hwaccelRows({
+        config,
+        hwaccel,
+        hwaccelFailed: !!hwaccelError,
+        stats,
+        t,
+      }),
+      enrichments: enrichmentRows({
+        config,
+        hardware,
+        probeFailed: !!probeError,
+        stats,
+        startup,
+        t,
+      }),
+      cameras: cameraConnectionCells(config, stats),
+    };
+  }, [config, hardware, probeError, hwaccel, hwaccelError, stats, t]);
+
+  return { rows, statsLoaded: !!stats };
+}

web/src/hooks/use-health-checks.ts

@@ -0,0 +1,192 @@
+import { useCallback, useSyncExternalStore } from "react";
+import axios from "axios";
+import useSWR, { useSWRConfig } from "swr";
+import type { TestResult } from "@/types/cameraWizard";
+import type { FrigateConfig } from "@/types/frigateConfig";
+import { ffprobeToTestResult, type FfprobeEntry } from "@/utils/streamIssues";
+import { activeCameras } from "@/utils/health";
+
+export type CameraStreamCheck = {
+  /** whole-camera failure (request error or timeout) */
+  error?: string;
+  /** one entry per config input, in config order */
+  streams: TestResult[];
+};
+
+export type StreamCheckResults = {
+  checkedAt: number;
+  byCamera: Record<string, CameraStreamCheck>;
+};
+
+type HealthChecksState = {
+  stream: {
+    results?: StreamCheckResults;
+    /** cameras still being probed in the current run */
+    pending: string[];
+    /** camera count of the current run */
+    total: number;
+  };
+  hardware: {
+    rechecking: boolean;
+    /** last on-demand probe; absent means the startup probe is current */
+    probedAt?: number;
+  };
+};
+
+// The tab bar button and both panes read this, so it lives outside React
+// and survives tab switches until reload. SWR is not used because a null
+// fetcher falls back to the global one and would GET /api/health/...
+let state: HealthChecksState = {
+  stream: { pending: [], total: 0 },
+  hardware: { rechecking: false },
+};
+const listeners = new Set<() => void>();
+let streamRunning = false;
+
+function subscribe(listener: () => void) {
+  listeners.add(listener);
+  return () => {
+    listeners.delete(listener);
+  };
+}
+
+function getState() {
+  return state;
+}
+
+function update(patch: (current: HealthChecksState) => HealthChecksState) {
+  state = patch(state);
+  listeners.forEach((listener) => listener());
+}
+
+const CONCURRENCY = 2;
+// the backend probes each input with a 6 s timeout plus one retry
+const TIMEOUT_PER_INPUT_MS = 12_000;
+const TIMEOUT_BASE_MS = 5_000;
+
+async function probeCamera(
+  name: string,
+  inputs: number,
+): Promise<CameraStreamCheck> {
+  try {
+    const response = await axios.get("ffprobe", {
+      params: { paths: `camera:${name}`, detailed: true },
+      timeout: TIMEOUT_BASE_MS + TIMEOUT_PER_INPUT_MS * Math.max(inputs, 1),
+    });
+    const entries: FfprobeEntry[] = Array.isArray(response.data)
+      ? response.data
+      : [];
+    return { streams: entries.map(ffprobeToTestResult) };
+  } catch (error) {
+    const axiosError = error as {
+      response?: { data?: { message?: string } };
+      message?: string;
+    };
+    return {
+      error:
+        axiosError.response?.data?.message ||
+        axiosError.message ||
+        "Connection failed",
+      streams: [],
+    };
+  }
+}
+
+async function runStreamChecks(config: FrigateConfig) {
+  if (streamRunning) {
+    return;
+  }
+
+  streamRunning = true;
+  const cameras = activeCameras(config);
+  const names = cameras.map((camera) => camera.name);
+  update((s) => ({
+    ...s,
+    stream: { ...s.stream, pending: names, total: names.length },
+  }));
+  const byCamera: Record<string, CameraStreamCheck> = {};
+  const queue = [...cameras];
+
+  const worker = async () => {
+    while (queue.length > 0) {
+      const camera = queue.shift();
+      if (!camera) {
+        return;
+      }
+      byCamera[camera.name] = await probeCamera(
+        camera.name,
+        camera.ffmpeg.inputs.length,
+      );
+      update((s) => ({
+        ...s,
+        stream: {
+          ...s.stream,
+          pending: s.stream.pending.filter((c) => c !== camera.name),
+        },
+      }));
+    }
+  };
+
+  try {
+    await Promise.all(
+      Array.from({ length: Math.min(CONCURRENCY, cameras.length) }, worker),
+    );
+    update((s) => ({
+      ...s,
+      stream: {
+        ...s.stream,
+        results: { checkedAt: Date.now() / 1000, byCamera },
+      },
+    }));
+  } finally {
+    streamRunning = false;
+  }
+}
+
+export function useHealthChecks() {
+  const { data: config } = useSWR<FrigateConfig>("config", {
+    revalidateOnFocus: false,
+  });
+  const { mutate } = useSWRConfig();
+  const current = useSyncExternalStore(subscribe, getState);
+
+  const run = useCallback(() => {
+    if (config) {
+      return runStreamChecks(config);
+    }
+  }, [config]);
+
+  const recheck = useCallback(async () => {
+    if (state.hardware.rechecking) {
+      return;
+    }
+
+    update((s) => ({ ...s, hardware: { ...s.hardware, rechecking: true } }));
+    try {
+      await axios.get("hardware/probe", { params: { refresh: true } });
+      await Promise.all([mutate("hardware/probe"), mutate("hardware/hwaccel")]);
+      update((s) => ({
+        ...s,
+        hardware: { rechecking: false, probedAt: Date.now() / 1000 },
+      }));
+    } catch {
+      update((s) => ({ ...s, hardware: { ...s.hardware, rechecking: false } }));
+    }
+  }, [mutate]);
+
+  const runAll = useCallback(
+    () => Promise.all([recheck(), run()]),
+    [recheck, run],
+  );
+
+  return {
+    stream: {
+      ...current.stream,
+      running: current.stream.pending.length > 0,
+      run,
+    },
+    hardware: { ...current.hardware, recheck },
+    runAll,
+    ready: !!config,
+  };
+}

web/src/hooks/use-stats.ts

@@ -4,7 +4,7 @@ import {
   CameraFfmpegThreshold,
   InferenceThreshold,
 } from "@/types/graph";
-import { FrigateStats, PotentialProblem } from "@/types/stats";
+import { FrigateStats, PotentialProblem, ProblemSeverity } from "@/types/stats";
 import { useMemo } from "react";
 import useSWR from "swr";
 import useDeepMemo from "./use-deep-memo";
@@ -15,6 +15,22 @@ import { useIsAdmin } from "./use-is-admin";
 
 import { useTranslation } from "react-i18next";
 
+// the status bar has always rendered these exact classes; keep them byte for
+// byte so its output does not change
+const SEVERITY_COLOR: Record<ProblemSeverity, string> = {
+  error: "text-danger",
+  warning: "text-orange-400",
+  info: "text-selected",
+};
+
+function problem(
+  severity: ProblemSeverity,
+  text: string,
+  relevantLink?: string,
+): PotentialProblem {
+  return { text, severity, color: SEVERITY_COLOR[severity], relevantLink };
+}
+
 export default function useStats(stats: FrigateStats | undefined) {
   const { t } = useTranslation(["views/system"]);
   const { data: config } = useSWR<FrigateConfig>("config");
@@ -48,36 +64,42 @@ export default function useStats(stats: FrigateStats | undefined) {
     // check shm level
     const shm = memoizedStats.service.storage["/dev/shm"];
     if (shm?.total && shm?.min_shm && shm.total < shm.min_shm) {
-      problems.push({
-        text: t("stats.shmTooLow", {
-          total: shm.total,
-          min: shm.min_shm,
-        }),
-        color: "text-danger",
-        relevantLink: "/system#storage",
-      });
+      problems.push(
+        problem(
+          "error",
+          t("stats.shmTooLow", {
+            total: shm.total,
+            min: shm.min_shm,
+          }),
+          "/system#storage",
+        ),
+      );
     }
 
     // check detectors for high inference speeds
     Object.entries(memoizedStats["detectors"]).forEach(([key, det]) => {
       if (det["inference_speed"] > InferenceThreshold.error) {
-        problems.push({
-          text: t("stats.detectIsVerySlow", {
-            detect: capitalizeFirstLetter(key),
-            speed: det["inference_speed"],
-          }),
-          color: "text-danger",
-          relevantLink: "/system#general",
-        });
+        problems.push(
+          problem(
+            "error",
+            t("stats.detectIsVerySlow", {
+              detect: capitalizeFirstLetter(key),
+              speed: det["inference_speed"],
+            }),
+            "/system#general",
+          ),
+        );
       } else if (det["inference_speed"] > InferenceThreshold.warning) {
-        problems.push({
-          text: t("stats.detectIsSlow", {
-            detect: capitalizeFirstLetter(key),
-            speed: det["inference_speed"],
-          }),
-          color: "text-orange-400",
-          relevantLink: "/system#general",
-        });
+        problems.push(
+          problem(
+            "warning",
+            t("stats.detectIsSlow", {
+              detect: capitalizeFirstLetter(key),
+              speed: det["inference_speed"],
+            }),
+            "/system#general",
+          ),
+        );
       }
     });
 
@@ -94,13 +116,15 @@ export default function useStats(stats: FrigateStats | undefined) {
 
       const cameraName = config.cameras?.[name]?.friendly_name ?? name;
       if (config.cameras?.[name]?.enabled && cam["camera_fps"] == 0) {
-        problems.push({
-          text: t("stats.cameraIsOffline", {
-            camera: capitalizeFirstLetter(capitalizeAll(cameraName)),
-          }),
-          color: "text-danger",
-          relevantLink: "logs",
-        });
+        problems.push(
+          problem(
+            "error",
+            t("stats.cameraIsOffline", {
+              camera: capitalizeFirstLetter(capitalizeAll(cameraName)),
+            }),
+            "logs",
+          ),
+        );
       }
     });
 
@@ -121,37 +145,43 @@ export default function useStats(stats: FrigateStats | undefined) {
       const cameraName = config?.cameras?.[name]?.friendly_name ?? name;
 
       if (!isNaN(ffmpegAvg) && ffmpegAvg >= CameraFfmpegThreshold.error) {
-        problems.push({
-          text: t("stats.ffmpegHighCpuUsage", {
-            camera: capitalizeFirstLetter(capitalizeAll(cameraName)),
-            ffmpegAvg,
-          }),
-          color: "text-danger",
-          relevantLink: "/system#cameras",
-        });
+        problems.push(
+          problem(
+            "error",
+            t("stats.ffmpegHighCpuUsage", {
+              camera: capitalizeFirstLetter(capitalizeAll(cameraName)),
+              ffmpegAvg,
+            }),
+            "/system#cameras",
+          ),
+        );
       }
 
       if (!isNaN(detectAvg) && detectAvg >= CameraDetectThreshold.error) {
-        problems.push({
-          text: t("stats.detectHighCpuUsage", {
-            camera: capitalizeFirstLetter(capitalizeAll(cameraName)),
-            detectAvg,
-          }),
-          color: "text-danger",
-          relevantLink: "/system#cameras",
-        });
+        problems.push(
+          problem(
+            "error",
+            t("stats.detectHighCpuUsage", {
+              camera: capitalizeFirstLetter(capitalizeAll(cameraName)),
+              detectAvg,
+            }),
+            "/system#cameras",
+          ),
+        );
       }
     });
 
     // Add message if debug replay is active
     if (replayActive) {
-      problems.push({
-        text: t("stats.debugReplayActive", {
-          defaultValue: "Debug replay session is active",
-        }),
-        color: "text-selected",
-        relevantLink: "/replay",
-      });
+      problems.push(
+        problem(
+          "info",
+          t("stats.debugReplayActive", {
+            defaultValue: "Debug replay session is active",
+          }),
+          "/replay",
+        ),
+      );
     }
 
     return problems;