retry and report Frigate+ connection failures at startup (#24545)

A Frigate+ model that wasn't cached yet needed api.frigate.video at startup, and when it couldn't be reached (a network that comes up late, a DNS blip) the requests ConnectionError wasn't a validation error, so Frigate crashed with a traceback before it could start. PlusApi requests now go through a session that retries connection failures for about 30 seconds, and a connection failure that outlasts that is raised as a ValueError so it shows up as a clear config validation error instead.

frigate/detectors/detector_config.py

@@ -186,15 +186,28 @@ class ModelConfig(BaseModel):
 
         # download the model if it doesn't exist
         if not os.path.isfile(self.path):
-            download_url = plus_api.get_model_download_url(model_id)
+            try:
-            r = requests.get(download_url)
+                download_url = plus_api.get_model_download_url(model_id)
+                r = requests.get(download_url)
+            except requests.exceptions.ConnectionError as e:
+                raise ValueError(
+                    f"Unable to connect to Frigate+ to download model {model_id}"
+                ) from e
+
             with open(self.path, "wb") as f:
                 f.write(r.content)
 
         # download the model info if it doesn't exist
         if not os.path.isfile(model_info_path):
+            try:
+                model_info = plus_api.get_model_info(model_id)
+            except requests.exceptions.ConnectionError as e:
+                raise ValueError(
+                    f"Unable to connect to Frigate+ to download model info for {model_id}"
+                ) from e
+
             with open(model_info_path, "w") as f:
-                json.dump(plus_api.get_model_info(model_id), f)
+                json.dump(model_info, f)
 
         model_info = load_plus_model_info(model_id)
 

frigate/plus.py

@@ -9,7 +9,9 @@ from typing import Any
 import cv2
 import requests
 from numpy import ndarray
+from requests.adapters import HTTPAdapter
 from requests.models import Response
+from urllib3.util.retry import Retry
 
 from frigate.const import MODEL_CACHE_DIR, PLUS_API_HOST, PLUS_ENV_VAR
 
@@ -101,6 +103,13 @@ class PlusApi:
         self._is_active: bool = self.key is not None
         self._token_data: dict = {}
 
+        # Retry connection failures so a network that comes up late at startup
+        # doesn't fail the Frigate+ model download
+        self._session = requests.Session()
+        self._session.mount(
+            self.host, HTTPAdapter(max_retries=Retry(connect=5, backoff_factor=1))
+        )
+
     def _refresh_token_if_needed(self) -> None:
         if (
             self._token_data.get("expires") is None
@@ -111,7 +120,9 @@ class PlusApi:
                     "Plus API key not set. See https://docs.frigate.video/integrations/plus#set-your-api-key"
                 )
             parts = self.key.split(":")
-            r = requests.get(f"{self.host}/v1/auth/token", auth=(parts[0], parts[1]))
+            r = self._session.get(
+                f"{self.host}/v1/auth/token", auth=(parts[0], parts[1])
+            )
             if not r.ok:
                 raise Exception(f"Unable to refresh API token: {r.text}")
             self._token_data = r.json()
@@ -121,19 +132,19 @@ class PlusApi:
         return {"authorization": f"Bearer {self._token_data.get('accessToken')}"}
 
     def _get(self, path: str) -> Response:
-        return requests.get(
+        return self._session.get(
             f"{self.host}/v1/{path}", headers=self._get_authorization_header()
         )
 
     def _post(self, path: str, data: dict) -> Response:
-        return requests.post(
+        return self._session.post(
             f"{self.host}/v1/{path}",
             headers=self._get_authorization_header(),
             json=data,
         )
 
     def _put(self, path: str, data: dict) -> Response:
-        return requests.put(
+        return self._session.put(
             f"{self.host}/v1/{path}",
             headers=self._get_authorization_header(),
             json=data,

frigate/test/test_config.py

@@ -6,6 +6,7 @@ from copy import deepcopy
 from unittest.mock import patch
 
 import numpy as np
+import requests
 from pydantic import ValidationError
 from ruamel.yaml.constructor import DuplicateKeyError
 
@@ -1595,6 +1596,31 @@ class TestConfig(unittest.TestCase):
         frigate_config = FrigateConfig(**config)
         assert frigate_config.primary_model.merged_labelmap[0] == "amazon"
 
+    @patch(
+        "frigate.plus.PlusApi.get_model_download_url",
+        side_effect=requests.exceptions.ConnectionError,
+    )
+    def test_plus_unreachable_is_validation_error(self, _):
+        config = {
+            "mqtt": {"host": "mqtt"},
+            "models": [{"path": "plus://unreachable", "devices": ["cpu"]}],
+            "cameras": {
+                "back": {
+                    "ffmpeg": {
+                        "inputs": [
+                            {
+                                "path": "rtsp://10.0.0.1:554/video",
+                                "roles": ["detect"],
+                            },
+                        ]
+                    },
+                }
+            },
+        }
+
+        with self.assertRaisesRegex(ValidationError, "Unable to connect to Frigate+"):
+            FrigateConfig(**config)
+
     def test_fails_on_invalid_role(self):
         config = {
             "mqtt": {"host": "mqtt"},