From 78eace258eec51e962bd6230601ce30ca8f5c829 Mon Sep 17 00:00:00 2001 From: Nicolas Mowen Date: Tue, 16 Dec 2025 21:35:43 -0700 Subject: [PATCH] Miscellaneous Fixes (0.17 Beta) (#21320) * Exclude D-FINE from using CUDA Graphs * fix objects count in detail stream * Add debugging for classification models * validate idb stored stream name and reset if invalid fixes https://github.com/blakeblackshear/frigate/discussions/21311 * ensure jina loading takes place in the main thread to prevent lazily importing tensorflow in another thread later reverts atexit changes in https://github.com/blakeblackshear/frigate/pull/21301 and fixes https://github.com/blakeblackshear/frigate/discussions/21306 * revert old atexit change in bird too * revert types * ensure we bail in the live mode hook for empty camera groups prevent infinite rendering on camera groups with no cameras --------- Co-authored-by: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> --- .../object_classification.md | 20 ++++++++++++ .../state_classification.md | 31 +++++++++++++++++++ frigate/data_processing/real_time/bird.py | 12 +++---- .../real_time/custom_classification.py | 14 ++++----- frigate/detectors/detection_runners.py | 1 + frigate/embeddings/maintainer.py | 23 -------------- frigate/embeddings/onnx/jina_v1_embedding.py | 3 ++ frigate/embeddings/onnx/jina_v2_embedding.py | 3 ++ web/src/components/timeline/DetailStream.tsx | 6 ++-- web/src/hooks/use-camera-live-mode.ts | 2 +- web/src/views/live/LiveCameraView.tsx | 22 ++++++++++--- 11 files changed, 93 insertions(+), 44 deletions(-) diff --git a/docs/docs/configuration/custom_classification/object_classification.md b/docs/docs/configuration/custom_classification/object_classification.md index 3d59b74f9..f94e5e0ec 100644 --- a/docs/docs/configuration/custom_classification/object_classification.md +++ b/docs/docs/configuration/custom_classification/object_classification.md @@ -94,3 +94,23 @@ When choosing which objects to classify, start with a small number of visually d - **Preprocessing**: Ensure examples reflect object crops similar to Frigate’s boxes; keep the subject centered. - **Labels**: Keep label names short and consistent; include a `none` class if you plan to ignore uncertain predictions for sub labels. - **Threshold**: Tune `threshold` per model to reduce false assignments. Start at `0.8` and adjust based on validation. + +## Debugging Classification Models + +To troubleshoot issues with object classification models, enable debug logging to see detailed information about classification attempts, scores, and consensus calculations. + +Enable debug logs for classification models by adding `frigate.data_processing.real_time.custom_classification: debug` to your `logger` configuration. These logs are verbose, so only keep this enabled when necessary. Restart Frigate after this change. + +```yaml +logger: + default: info + logs: + frigate.data_processing.real_time.custom_classification: debug +``` + +The debug logs will show: + +- Classification probabilities for each attempt +- Whether scores meet the threshold requirement +- Consensus calculations and when assignments are made +- Object classification history and weighted scores diff --git a/docs/docs/configuration/custom_classification/state_classification.md b/docs/docs/configuration/custom_classification/state_classification.md index 801d5d905..2b7d16da1 100644 --- a/docs/docs/configuration/custom_classification/state_classification.md +++ b/docs/docs/configuration/custom_classification/state_classification.md @@ -70,3 +70,34 @@ Once some images are assigned, training will begin automatically. - **Data collection**: Use the model's Recent Classifications tab to gather balanced examples across times of day and weather. - **When to train**: Focus on cases where the model is entirely incorrect or flips between states when it should not. There's no need to train additional images when the model is already working consistently. - **Selecting training images**: Images scoring below 100% due to new conditions (e.g., first snow of the year, seasonal changes) or variations (e.g., objects temporarily in view, insects at night) are good candidates for training, as they represent scenarios different from the default state. Training these lower-scoring images that differ from existing training data helps prevent overfitting. Avoid training large quantities of images that look very similar, especially if they already score 100% as this can lead to overfitting. + +## Debugging Classification Models + +To troubleshoot issues with state classification models, enable debug logging to see detailed information about classification attempts, scores, and state verification. + +Enable debug logs for classification models by adding `frigate.data_processing.real_time.custom_classification: debug` to your `logger` configuration. These logs are verbose, so only keep this enabled when necessary. Restart Frigate after this change. + +```yaml +logger: + default: info + logs: + frigate.data_processing.real_time.custom_classification: debug +``` + +The debug logs will show: + +- Classification probabilities for each attempt +- Whether scores meet the threshold requirement +- State verification progress (consecutive detections needed) +- When state changes are published + +### Recent Classifications + +For state classification, images are only added to recent classifications under specific circumstances: + +- **First detection**: The first classification attempt for a camera is always saved +- **State changes**: Images are saved when the detected state differs from the current verified state +- **Pending verification**: Images are saved when there's a pending state change being verified (requires 3 consecutive identical states) +- **Low confidence**: Images with scores below 100% are saved even if the state matches the current state (useful for training) + +Images are **not** saved when the state is stable (detected state matches current state) **and** the score is 100%. This prevents unnecessary storage of redundant high-confidence classifications. diff --git a/frigate/data_processing/real_time/bird.py b/frigate/data_processing/real_time/bird.py index 8d6e1b2dc..e599ab0fb 100644 --- a/frigate/data_processing/real_time/bird.py +++ b/frigate/data_processing/real_time/bird.py @@ -19,6 +19,11 @@ from frigate.util.object import calculate_region from ..types import DataProcessorMetrics from .api import RealTimeProcessorApi +try: + from tflite_runtime.interpreter import Interpreter +except ModuleNotFoundError: + from tensorflow.lite.python.interpreter import Interpreter + logger = logging.getLogger(__name__) @@ -30,7 +35,7 @@ class BirdRealTimeProcessor(RealTimeProcessorApi): metrics: DataProcessorMetrics, ): super().__init__(config, metrics) - self.interpreter: Any | None = None + self.interpreter: Interpreter = None self.sub_label_publisher = sub_label_publisher self.tensor_input_details: dict[str, Any] = None self.tensor_output_details: dict[str, Any] = None @@ -77,11 +82,6 @@ class BirdRealTimeProcessor(RealTimeProcessorApi): @redirect_output_to_logger(logger, logging.DEBUG) def __build_detector(self) -> None: - try: - from tflite_runtime.interpreter import Interpreter - except ModuleNotFoundError: - from tensorflow.lite.python.interpreter import Interpreter - self.interpreter = Interpreter( model_path=os.path.join(MODEL_CACHE_DIR, "bird/bird.tflite"), num_threads=2, diff --git a/frigate/data_processing/real_time/custom_classification.py b/frigate/data_processing/real_time/custom_classification.py index dd011b48e..a2f88ee93 100644 --- a/frigate/data_processing/real_time/custom_classification.py +++ b/frigate/data_processing/real_time/custom_classification.py @@ -29,6 +29,11 @@ from frigate.util.object import box_overlaps, calculate_region from ..types import DataProcessorMetrics from .api import RealTimeProcessorApi +try: + from tflite_runtime.interpreter import Interpreter +except ModuleNotFoundError: + from tensorflow.lite.python.interpreter import Interpreter + logger = logging.getLogger(__name__) MAX_OBJECT_CLASSIFICATIONS = 16 @@ -47,7 +52,7 @@ class CustomStateClassificationProcessor(RealTimeProcessorApi): self.requestor = requestor self.model_dir = os.path.join(MODEL_CACHE_DIR, self.model_config.name) self.train_dir = os.path.join(CLIPS_DIR, self.model_config.name, "train") - self.interpreter: Any | None = None + self.interpreter: Interpreter = None self.tensor_input_details: dict[str, Any] | None = None self.tensor_output_details: dict[str, Any] | None = None self.labelmap: dict[int, str] = {} @@ -345,7 +350,7 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): self.model_config = model_config self.model_dir = os.path.join(MODEL_CACHE_DIR, self.model_config.name) self.train_dir = os.path.join(CLIPS_DIR, self.model_config.name, "train") - self.interpreter: Any | None = None + self.interpreter: Interpreter = None self.sub_label_publisher = sub_label_publisher self.requestor = requestor self.tensor_input_details: dict[str, Any] | None = None @@ -368,11 +373,6 @@ class CustomObjectClassificationProcessor(RealTimeProcessorApi): @redirect_output_to_logger(logger, logging.DEBUG) def __build_detector(self) -> None: - try: - from tflite_runtime.interpreter import Interpreter - except ModuleNotFoundError: - from tensorflow.lite.python.interpreter import Interpreter - model_path = os.path.join(self.model_dir, "model.tflite") labelmap_path = os.path.join(self.model_dir, "labelmap.txt") diff --git a/frigate/detectors/detection_runners.py b/frigate/detectors/detection_runners.py index 89ebb35eb..56b49ec67 100644 --- a/frigate/detectors/detection_runners.py +++ b/frigate/detectors/detection_runners.py @@ -170,6 +170,7 @@ class CudaGraphRunner(BaseModelRunner): return model_type not in [ ModelTypeEnum.yolonas.value, + ModelTypeEnum.dfine.value, EnrichmentModelTypeEnum.paddleocr.value, EnrichmentModelTypeEnum.jina_v1.value, EnrichmentModelTypeEnum.jina_v2.value, diff --git a/frigate/embeddings/maintainer.py b/frigate/embeddings/maintainer.py index 33d09dcc3..78a251c42 100644 --- a/frigate/embeddings/maintainer.py +++ b/frigate/embeddings/maintainer.py @@ -146,29 +146,6 @@ class EmbeddingMaintainer(threading.Thread): self.detected_license_plates: dict[str, dict[str, Any]] = {} self.genai_client = get_genai_client(config) - # Pre-import TensorFlow/tflite on main thread to avoid atexit registration issues - # when importing from worker threads later (e.g., during dynamic config updates) - if ( - self.config.classification.bird.enabled - or len(self.config.classification.custom) > 0 - ): - try: - from tflite_runtime.interpreter import Interpreter # noqa: F401 - except ModuleNotFoundError: - try: - from tensorflow.lite.python.interpreter import ( # noqa: F401 - Interpreter, - ) - - logger.debug( - "Pre-imported TensorFlow Interpreter on main thread for classification models" - ) - except Exception as e: - logger.warning( - f"Failed to pre-import TensorFlow Interpreter: {e}. " - "Classification models may fail to load if added dynamically." - ) - # model runners to share between realtime and post processors if self.config.lpr.enabled: lpr_model_runner = LicensePlateModelRunner( diff --git a/frigate/embeddings/onnx/jina_v1_embedding.py b/frigate/embeddings/onnx/jina_v1_embedding.py index e64d8da39..519247f3c 100644 --- a/frigate/embeddings/onnx/jina_v1_embedding.py +++ b/frigate/embeddings/onnx/jina_v1_embedding.py @@ -186,6 +186,9 @@ class JinaV1ImageEmbedding(BaseEmbedding): download_func=self._download_model, ) self.downloader.ensure_model_files() + # Avoid lazy loading in worker threads: block until downloads complete + # and load the model on the main thread during initialization. + self._load_model_and_utils() else: self.downloader = None ModelDownloader.mark_files_state( diff --git a/frigate/embeddings/onnx/jina_v2_embedding.py b/frigate/embeddings/onnx/jina_v2_embedding.py index 44cc6c12b..fd4323f85 100644 --- a/frigate/embeddings/onnx/jina_v2_embedding.py +++ b/frigate/embeddings/onnx/jina_v2_embedding.py @@ -65,6 +65,9 @@ class JinaV2Embedding(BaseEmbedding): download_func=self._download_model, ) self.downloader.ensure_model_files() + # Avoid lazy loading in worker threads: block until downloads complete + # and load the model on the main thread during initialization. + self._load_model_and_utils() else: self.downloader = None ModelDownloader.mark_files_state( diff --git a/web/src/components/timeline/DetailStream.tsx b/web/src/components/timeline/DetailStream.tsx index dc87e1f0f..e28c841b1 100644 --- a/web/src/components/timeline/DetailStream.tsx +++ b/web/src/components/timeline/DetailStream.tsx @@ -345,9 +345,9 @@ function ReviewGroup({ } const reviewInfo = useMemo(() => { - const objectCount = fetchedEvents - ? fetchedEvents.length - : (review.data.objects ?? []).length; + const detectionsCount = + review.data?.detections?.length ?? (review.data?.objects ?? []).length; + const objectCount = fetchedEvents ? fetchedEvents.length : detectionsCount; return `${t("detail.trackedObject", { count: objectCount })}`; }, [review, t, fetchedEvents]); diff --git a/web/src/hooks/use-camera-live-mode.ts b/web/src/hooks/use-camera-live-mode.ts index 0b189a1f2..288c0ea09 100644 --- a/web/src/hooks/use-camera-live-mode.ts +++ b/web/src/hooks/use-camera-live-mode.ts @@ -54,7 +54,7 @@ export default function useCameraLiveMode( }>({}); useEffect(() => { - if (!cameras) return; + if (!cameras || cameras.length === 0) return; const mseSupported = "MediaSource" in window || "ManagedMediaSource" in window; diff --git a/web/src/views/live/LiveCameraView.tsx b/web/src/views/live/LiveCameraView.tsx index a9c62f623..5de52d243 100644 --- a/web/src/views/live/LiveCameraView.tsx +++ b/web/src/views/live/LiveCameraView.tsx @@ -147,10 +147,11 @@ export default function LiveCameraView({ // supported features - const [streamName, setStreamName] = useUserPersistence( - `${camera.name}-stream`, - Object.values(camera.live.streams)[0], - ); + const [streamName, setStreamName, streamNameLoaded] = + useUserPersistence( + `${camera.name}-stream`, + Object.values(camera.live.streams)[0], + ); const isRestreamed = useMemo( () => @@ -159,6 +160,19 @@ export default function LiveCameraView({ [config, streamName], ); + // validate stored stream name and reset if now invalid + + useEffect(() => { + if (!streamNameLoaded) return; + + const available = Object.values(camera.live.streams || {}); + if (available.length === 0) return; + + if (streamName != null && !available.includes(streamName)) { + setStreamName(available[0]); + } + }, [streamNameLoaded, camera.live.streams, streamName, setStreamName]); + const { data: cameraMetadata } = useSWR( isRestreamed ? `go2rtc/streams/${streamName}` : null, {