retry and report Frigate+ connection failures at startup

A Frigate+ model that wasn't cached yet needed api.frigate.video at startup, and when it couldn't be reached (a network that comes up late, a DNS blip) the requests ConnectionError wasn't a validation error, so Frigate crashed with a traceback before it could start. PlusApi requests now go through a session that retries connection failures for about 30 seconds, and a connection failure that outlasts that is raised as a ValueError so it shows up as a clear config validation error instead.

docs/docs/configuration/advanced/reference.md

@@ -571,8 +571,6 @@ notifications:
   enabled: False
   # Optional: Email for push service to reach out to
   # NOTE: This is required to use notifications
-  # NOTE: Email can be specified with an environment variable or docker secrets that must begin with 'FRIGATE_'.
-  #       e.g. email: '{FRIGATE_NOTIFICATION_EMAIL}'
   email: "admin@example.com"
   # Optional: Cooldown time for notifications in seconds (default: shown below)
   cooldown: 0

frigate/api/app.py

@@ -311,12 +311,9 @@ def config(request: Request):
         mode="json", warnings="none", exclude_none=True
     )
 
-    is_admin = request.headers.get("remote-role") == "admin"
-
-    # hide environment_vars and the notification email from non-admin users
-    if not is_admin:
+    # remove environment_vars for non-admin users
+    if request.headers.get("remote-role") != "admin":
         config.pop("environment_vars", None)
-        redact_credential(config["notifications"], "email")
 
     # redact mqtt credentials
     redact_credential(config["mqtt"], "password")
@@ -373,15 +370,7 @@ def config(request: Request):
                 camera_name
             )
             if base_sections:
-                # copy so redaction below can't alter the profile manager's cache
-                camera_dict["base_config"] = copy.deepcopy(base_sections)
-
-        # cameras inherit the global notification email
-        if not is_admin:
-            redact_credential(camera_dict["notifications"], "email")
-            redact_credential(
-                camera_dict.get("base_config", {}).get("notifications", {}), "email"
-            )
+                camera_dict["base_config"] = base_sections
 
     # remove go2rtc stream passwords
     go2rtc: dict[str, Any] = config_obj.go2rtc.model_dump(

frigate/config/camera/notification.py

@@ -1,7 +1,6 @@
 from pydantic import Field
 
 from ..base import FrigateBaseModel
-from ..env import EnvString
 
 __all__ = ["NotificationConfig"]
 
@@ -12,7 +11,7 @@ class NotificationConfig(FrigateBaseModel):
         title="Enable notifications",
         description="Enable or disable notifications for all cameras; can be overridden per-camera.",
     )
-    email: EnvString | None = Field(
+    email: str | None = Field(
         default=None,
         title="Notification email",
         description="Email address used for push notifications or required by certain notification providers.",

frigate/detectors/detector_config.py

@@ -186,15 +186,28 @@ class ModelConfig(BaseModel):
 
         # download the model if it doesn't exist
         if not os.path.isfile(self.path):
-            download_url = plus_api.get_model_download_url(model_id)
-            r = requests.get(download_url)
+            try:
+                download_url = plus_api.get_model_download_url(model_id)
+                r = requests.get(download_url)
+            except requests.exceptions.ConnectionError as e:
+                raise ValueError(
+                    f"Unable to connect to Frigate+ to download model {model_id}"
+                ) from e
+
             with open(self.path, "wb") as f:
                 f.write(r.content)
 
         # download the model info if it doesn't exist
         if not os.path.isfile(model_info_path):
+            try:
+                model_info = plus_api.get_model_info(model_id)
+            except requests.exceptions.ConnectionError as e:
+                raise ValueError(
+                    f"Unable to connect to Frigate+ to download model info for {model_id}"
+                ) from e
+
             with open(model_info_path, "w") as f:
-                json.dump(plus_api.get_model_info(model_id), f)
+                json.dump(model_info, f)
 
         model_info = load_plus_model_info(model_id)
 

frigate/plus.py

@@ -9,7 +9,9 @@ from typing import Any
 import cv2
 import requests
 from numpy import ndarray
+from requests.adapters import HTTPAdapter
 from requests.models import Response
+from urllib3.util.retry import Retry
 
 from frigate.const import MODEL_CACHE_DIR, PLUS_API_HOST, PLUS_ENV_VAR
 
@@ -101,6 +103,13 @@ class PlusApi:
         self._is_active: bool = self.key is not None
         self._token_data: dict = {}
 
+        # Retry connection failures so a network that comes up late at startup
+        # doesn't fail the Frigate+ model download
+        self._session = requests.Session()
+        self._session.mount(
+            self.host, HTTPAdapter(max_retries=Retry(connect=5, backoff_factor=1))
+        )
+
     def _refresh_token_if_needed(self) -> None:
         if (
             self._token_data.get("expires") is None
@@ -111,7 +120,9 @@ class PlusApi:
                     "Plus API key not set. See https://docs.frigate.video/integrations/plus#set-your-api-key"
                 )
             parts = self.key.split(":")
-            r = requests.get(f"{self.host}/v1/auth/token", auth=(parts[0], parts[1]))
+            r = self._session.get(
+                f"{self.host}/v1/auth/token", auth=(parts[0], parts[1])
+            )
             if not r.ok:
                 raise Exception(f"Unable to refresh API token: {r.text}")
             self._token_data = r.json()
@@ -121,19 +132,19 @@ class PlusApi:
         return {"authorization": f"Bearer {self._token_data.get('accessToken')}"}
 
     def _get(self, path: str) -> Response:
-        return requests.get(
+        return self._session.get(
             f"{self.host}/v1/{path}", headers=self._get_authorization_header()
         )
 
     def _post(self, path: str, data: dict) -> Response:
-        return requests.post(
+        return self._session.post(
             f"{self.host}/v1/{path}",
             headers=self._get_authorization_header(),
             json=data,
         )
 
     def _put(self, path: str, data: dict) -> Response:
-        return requests.put(
+        return self._session.put(
             f"{self.host}/v1/{path}",
             headers=self._get_authorization_header(),
             json=data,

frigate/stats/util.py

@@ -236,15 +236,6 @@ def skipped_percent(skipped_fps: float, camera_fps: float, enabled: bool) -> flo
     return round(skipped_fps / camera_fps * 100, 1)
 
 
-def get_go2rtc_pid(cpu_usages: dict[str, dict[str, Any]]) -> int | None:
-    """Find the pid of the running go2rtc process in the cpu usages."""
-    for pid, usage in cpu_usages.items():
-        if usage.get("cmdline", "").split(" ")[0].endswith("/go2rtc"):
-            return int(pid)
-
-    return None
-
-
 def stats_snapshot(
     config: FrigateConfig,
     stats_tracking: StatsTrackingTypes,
@@ -365,14 +356,6 @@ def stats_snapshot(
 
     stats["service"]["storage"]["/dev/shm"] = calculate_shm_requirements(config)
 
-    cpu_usages = stats.get("cpu_usages", {})
-
-    # go2rtc is supervised by s6, so its pid changes when s6 restarts it
-    go2rtc_pid = get_go2rtc_pid(cpu_usages)
-
-    if go2rtc_pid is not None:
-        stats_tracking["processes"]["go2rtc"] = go2rtc_pid
-
     stats["processes"] = {}
     for name, pid in stats_tracking["processes"].items():
         stats["processes"][name] = {
@@ -381,6 +364,7 @@ def stats_snapshot(
 
     # Embed cpu/mem stats into detectors, cameras, and processes
     # so history consumers don't need the full cpu_usages dict
+    cpu_usages = stats.get("cpu_usages", {})
 
     for det_stats in stats["detectors"].values():
         pid_str = str(det_stats.get("pid", ""))

frigate/test/http_api/test_http_app.py

@@ -4,7 +4,6 @@ from unittest.mock import Mock, patch
 
 import frigate.genai
 from frigate.config import GenAIProviderEnum
-from frigate.config.env import FRIGATE_ENV_VARS
 from frigate.const import MODEL_CACHE_DIR, REDACTED_CREDENTIAL_SENTINEL
 from frigate.genai import GenAIClient
 from frigate.models import Event, Recordings, ReviewSegment
@@ -112,30 +111,6 @@ class TestHttpApp(BaseTestHttp):
             mqtt = response.json()["mqtt"]
             assert mqtt["password"] == REDACTED_CREDENTIAL_SENTINEL
 
-    def test_config_response_hides_notification_email_from_viewers(self):
-        self.minimal_config["notifications"] = {"email": "{FRIGATE_TEST_EMAIL}"}
-
-        with patch.dict(FRIGATE_ENV_VARS, {"FRIGATE_TEST_EMAIL": "me@example.com"}):
-            app = super().create_app()
-
-        assert app.frigate_config.notifications.email == "me@example.com"
-
-        with AuthTestClient(app) as client:
-            response = client.get(
-                "/config",
-                headers={"remote-user": "viewer", "remote-role": "viewer"},
-            )
-            assert response.status_code == 200
-            config = response.json()
-            assert config["notifications"]["email"] == REDACTED_CREDENTIAL_SENTINEL
-            assert (
-                config["cameras"]["front_door"]["notifications"]["email"]
-                == REDACTED_CREDENTIAL_SENTINEL
-            )
-
-            response = client.get("/config")
-            assert response.json()["notifications"]["email"] == "me@example.com"
-
     def test_config_response_keeps_plus_model_reference(self):
         model_id = "test_plus_reference"
         model_path = os.path.join(MODEL_CACHE_DIR, model_id)

frigate/test/test_config.py

@@ -6,6 +6,7 @@ from copy import deepcopy
 from unittest.mock import patch
 
 import numpy as np
+import requests
 from pydantic import ValidationError
 from ruamel.yaml.constructor import DuplicateKeyError
 
@@ -1595,6 +1596,31 @@ class TestConfig(unittest.TestCase):
         frigate_config = FrigateConfig(**config)
         assert frigate_config.primary_model.merged_labelmap[0] == "amazon"
 
+    @patch(
+        "frigate.plus.PlusApi.get_model_download_url",
+        side_effect=requests.exceptions.ConnectionError,
+    )
+    def test_plus_unreachable_is_validation_error(self, _):
+        config = {
+            "mqtt": {"host": "mqtt"},
+            "models": [{"path": "plus://unreachable", "devices": ["cpu"]}],
+            "cameras": {
+                "back": {
+                    "ffmpeg": {
+                        "inputs": [
+                            {
+                                "path": "rtsp://10.0.0.1:554/video",
+                                "roles": ["detect"],
+                            },
+                        ]
+                    },
+                }
+            },
+        }
+
+        with self.assertRaisesRegex(ValidationError, "Unable to connect to Frigate+"):
+            FrigateConfig(**config)
+
     def test_fails_on_invalid_role(self):
         config = {
             "mqtt": {"host": "mqtt"},

frigate/test/test_go2rtc_pid_stats.py

@@ -1,70 +0,0 @@
-"""Tests for resolving the go2rtc pid from cpu usages."""
-
-import unittest
-from types import SimpleNamespace
-from unittest.mock import Mock, patch
-
-from frigate.stats.util import get_go2rtc_pid, stats_snapshot
-
-
-class TestGo2rtcPid(unittest.TestCase):
-    def test_finds_go2rtc_by_binary_path(self):
-        cpu_usages = {
-            "frigate.full_system": {"cpu": "1.0", "mem": "2.0"},
-            "100": {"cmdline": "ffmpeg -i rtsp://127.0.0.1:8554/go2rtc_cam"},
-            "200": {
-                "cmdline": "/usr/local/go2rtc/bin/go2rtc -config=/dev/shm/go2rtc.yaml"
-            },
-            "300": {"cmdline": "frigate.recording"},
-        }
-
-        self.assertEqual(get_go2rtc_pid(cpu_usages), 200)
-
-    def test_finds_custom_go2rtc_binary(self):
-        self.assertEqual(get_go2rtc_pid({"42": {"cmdline": "/config/go2rtc"}}), 42)
-
-    def test_returns_none_when_go2rtc_is_not_running(self):
-        self.assertIsNone(get_go2rtc_pid({"100": {"cmdline": "ffmpeg -i x"}}))
-        self.assertIsNone(get_go2rtc_pid({}))
-
-
-class TestGo2rtcPidInSnapshot(unittest.TestCase):
-    def snapshot(self, tracking: dict, go2rtc_pid: int) -> dict:
-        def update_stats(stats: dict) -> None:
-            stats["cpu_usages"] = {
-                str(go2rtc_pid): {
-                    "cmdline": "/usr/local/go2rtc/bin/go2rtc -config=x",
-                    "cpu": str(go2rtc_pid / 100),
-                    "mem": str(go2rtc_pid / 10),
-                }
-            }
-
-        config = SimpleNamespace(
-            cameras={},
-            telemetry=SimpleNamespace(stats=SimpleNamespace(network_bandwidth=False)),
-        )
-        hardware_stats = Mock()
-        hardware_stats.update_stats.side_effect = update_stats
-
-        with (
-            patch("frigate.stats.util.get_detector_stats", return_value={}),
-            patch("frigate.stats.util.embeddings_stats", return_value={}),
-            patch("frigate.stats.util.calculate_shm_requirements", return_value={}),
-        ):
-            return stats_snapshot(config, tracking, hardware_stats)
-
-    def test_snapshot_follows_go2rtc_restart(self):
-        tracking = {
-            "camera_metrics": {},
-            "detectors": {},
-            "started": 0,
-            "latest_frigate_version": "",
-            "processes": {"go2rtc": 200, "recording": 50},
-            "storage_maintainer": None,
-        }
-
-        first = self.snapshot(tracking, 200)["processes"]["go2rtc"]
-        self.assertEqual(first, {"pid": 200, "cpu": "2.0", "mem": "20.0"})
-
-        restarted = self.snapshot(tracking, 300)["processes"]["go2rtc"]
-        self.assertEqual(restarted, {"pid": 300, "cpu": "3.0", "mem": "30.0"})