diff --git a/frigate/analytics/collectors/detection.py b/frigate/analytics/collectors/detection.py new file mode 100644 index 0000000000..1815b3e74d --- /dev/null +++ b/frigate/analytics/collectors/detection.py @@ -0,0 +1,65 @@ +"""Detection section: models, the detectors they run on, and inference speed.""" + +import os + +from frigate.analytics.collectors.common import rate +from frigate.analytics.context import ReportContext +from frigate.analytics.schema import DetectionModel, DetectionSection, ModelSource +from frigate.config.config import DEFAULT_MODEL +from frigate.const import MODEL_CACHE_DIR +from frigate.detectors.detector_config import SceneEnum +from frigate.detectors.detector_types import DetectorTypeEnum +from frigate.detectors.device import runner_names + +# the paths FrigateConfig fills in for a model that sets none +BUNDLED_MODEL_PATHS = frozenset( + {"/cpu_model.tflite", "/edgetpu_model.tflite", str(DEFAULT_MODEL["path"])} +) + + +def model_source(path: str | None) -> ModelSource: + """Default, Frigate+ (a cached model next to its info file), or custom. + + Parsing rewrites plus:// to the model cache, so the prefix is gone by now. + """ + if path is None or path in BUNDLED_MODEL_PATHS: + return ModelSource.default + + if path.startswith(f"{MODEL_CACHE_DIR}/") and os.path.isfile(f"{path}.json"): + return ModelSource.plus + + return ModelSource.custom + + +def collect(ctx: ReportContext) -> DetectionSection: + config = ctx.config + detectors = ctx.stats.get("detectors", {}) + model_specs = [(model, config.devices_for_model(model)) for model in config.models] + + # FrigateApp.start_detectors names the processes in this same order + names = iter(runner_names([spec for _, specs in model_specs for spec in specs])) + models: dict[SceneEnum, DetectionModel] = {} + + for model, specs in model_specs: + speeds: list[float] = [] + + for _ in specs: + speed = detectors.get(next(names), {}).get("inference_speed") + + if isinstance(speed, int | float) and speed > 0: + speeds.append(float(speed)) + + models[model.scene] = DetectionModel( + detector=DetectorTypeEnum(specs[0].detector), + devices=len(specs), + model_type=model.model_type, + input=f"{model.width}x{model.height}", + source=model_source(model.path), + inference_ms=rate(sum(speeds) / len(speeds)) if speeds else None, + ) + + return DetectionSection( + models=models, + detection_fps=rate(ctx.stats.get("detection_fps")), + skipped_fps=rate(ctx.stats.get("skipped_fps")), + ) diff --git a/frigate/test/test_analytics_collectors_detection.py b/frigate/test/test_analytics_collectors_detection.py new file mode 100644 index 0000000000..d1601e128b --- /dev/null +++ b/frigate/test/test_analytics_collectors_detection.py @@ -0,0 +1,61 @@ +"""Tests for the detection collector.""" + +import os +import tempfile +import unittest +from unittest.mock import patch + +from frigate.analytics.collectors import detection +from frigate.analytics.schema import ModelSource +from frigate.detectors.detector_config import SceneEnum +from frigate.test.analytics_helpers import make_config, make_context + + +class TestDetectionCollector(unittest.TestCase): + def test_reports_each_model_with_its_mean_inference_speed(self): + config = make_config({"models": [{"devices": ["cpu", "cpu"]}]}) + stats = { + "detectors": { + "cpu": {"inference_speed": 10.0}, + "cpu#2": {"inference_speed": 20.0}, + }, + "detection_fps": 12.346, + "skipped_fps": 0, + } + + section = detection.collect(make_context(config, stats)) + + model = section.models[SceneEnum.all] + self.assertEqual(model.detector, "cpu") + self.assertEqual(model.devices, 2) + self.assertEqual(model.model_type, "ssd") + self.assertEqual(model.input, "320x320") + self.assertEqual(model.source, ModelSource.default) + self.assertEqual(model.inference_ms, 15.0) + self.assertEqual(section.detection_fps, 12.35) + self.assertEqual(section.skipped_fps, 0.0) + + def test_inference_is_null_before_the_first_stats(self): + section = detection.collect(make_context()) + + self.assertIsNone(section.models[SceneEnum.all].inference_ms) + + def test_model_source(self): + with tempfile.TemporaryDirectory() as cache: + plus_model = os.path.join(cache, "abc123") + open(f"{plus_model}.json", "w").close() + + with patch.object(detection, "MODEL_CACHE_DIR", cache): + self.assertEqual(detection.model_source(plus_model), ModelSource.plus) + self.assertEqual( + detection.model_source(os.path.join(cache, "no_info")), + ModelSource.custom, + ) + + self.assertEqual(detection.model_source(None), ModelSource.default) + self.assertEqual( + detection.model_source("/cpu_model.tflite"), ModelSource.default + ) + self.assertEqual( + detection.model_source("/config/yolo.onnx"), ModelSource.custom + )