diff --git a/frigate/analytics/collectors/features.py b/frigate/analytics/collectors/features.py new file mode 100644 index 0000000000..a0247c98c3 --- /dev/null +++ b/frigate/analytics/collectors/features.py @@ -0,0 +1,145 @@ +"""Features section: enrichments, GenAI, integrations, and users.""" + +from collections import Counter + +from frigate.analytics.collectors.common import closed, histogram +from frigate.analytics.context import ReportContext +from frigate.analytics.schema import ( + BirdseyeUsage, + ClassificationUsage, + EnrichmentDevice, + EnrichmentUsage, + FeaturesSection, + GenAIUsage, + SemanticSearchModel, + SemanticSearchUsage, + TranscriptionModel, + TranscriptionUsage, + UserRole, +) +from frigate.config.camera.genai import GenAIProviderEnum, GenAIRoleEnum +from frigate.const import REPLAY_CAMERA_PREFIX +from frigate.models import User + +RUNTIME_DEVICES = { + "cpu": EnrichmentDevice.cpu, + "cuda": EnrichmentDevice.cuda, + "tensorrt": EnrichmentDevice.tensorrt, + "migraphx": EnrichmentDevice.migraphx, +} +OPENVINO_DEVICES = { + "cpu": EnrichmentDevice.openvino_cpu, + "gpu": EnrichmentDevice.openvino_gpu, + "npu": EnrichmentDevice.openvino_npu, +} + + +def enrichment_device(label: object) -> EnrichmentDevice | None: + """Map a runner's device label, like "CUDA" or "OpenVINO GPU.0,CPU".""" + if not isinstance(label, str) or not label: + return None + + runtime, _, target = label.partition(" ") + + if runtime == "OpenVINO": + first = target.split(",")[0].split(".")[0].strip().lower() + return OPENVINO_DEVICES.get(first, EnrichmentDevice.other) + + return RUNTIME_DEVICES.get(label.lower(), EnrichmentDevice.other) + + +def model_name(model: object) -> str | None: + if model is None: + return None + + return str(getattr(model, "value", model)) + + +def semantic_model(model: object) -> SemanticSearchModel | None: + # any string that isn't a built-in model names a GenAI provider + name = model_name(model) + + if name is None: + return None + + if name in ("jinav1", "jinav2"): + return SemanticSearchModel(name) + + return SemanticSearchModel.genai + + +def transcription_model(model: object) -> TranscriptionModel | None: + name = model_name(model) + + if name is None: + return None + + return TranscriptionModel.whisper if name == "whisper" else TranscriptionModel.genai + + +def users() -> dict[UserRole, int]: + roles: Counter[UserRole] = Counter( + closed(UserRole, user.role, UserRole.custom) for user in User.select(User.role) + ) + return histogram(roles) + + +def collect(ctx: ReportContext) -> FeaturesSection: + config = ctx.config + devices = ctx.stats.get("embeddings", {}).get("devices", {}) + cameras = [ + camera + for name, camera in config.cameras.items() + if not name.startswith(REPLAY_CAMERA_PREFIX) + ] + providers: Counter[GenAIProviderEnum] = Counter( + genai.provider for genai in config.genai.values() + ) + roles: Counter[GenAIRoleEnum] = Counter( + role for genai in config.genai.values() for role in genai.roles + ) + custom = list(config.classification.custom.values()) + + return FeaturesSection( + face_recognition=EnrichmentUsage( + enabled=config.face_recognition.enabled, + model_size=config.face_recognition.model_size, + device=enrichment_device(devices.get("face_recognition")), + ), + lpr=EnrichmentUsage( + enabled=config.lpr.enabled, + model_size=config.lpr.model_size, + device=enrichment_device(devices.get("lpr")), + ), + semantic_search=SemanticSearchUsage( + enabled=config.semantic_search.enabled, + model=semantic_model(config.semantic_search.model), + model_size=config.semantic_search.model_size, + device=enrichment_device(devices.get("semantic_search")), + triggers=sum(len(camera.semantic_search.triggers) for camera in cameras), + ), + audio_transcription=TranscriptionUsage( + enabled=config.audio_transcription.enabled, + model=transcription_model(config.audio_transcription.model), + model_size=config.audio_transcription.model_size, + ), + genai=GenAIUsage(providers=histogram(providers), roles=histogram(roles)), + classification_models=ClassificationUsage( + state=sum(1 for model in custom if model.state_config is not None), + object=sum(1 for model in custom if model.object_config is not None), + ), + birdseye=BirdseyeUsage( + enabled=config.birdseye.enabled, + modes=list(config.birdseye.modes), + restream=config.birdseye.restream, + ), + mqtt=config.mqtt.enabled, + notifications=config.notifications.enabled, + auth=config.auth.enabled, + proxy_auth=config.proxy.header_map.user is not None, + tls=config.tls.enabled, + users=users(), + camera_groups=len(config.camera_groups), + profiles=len(config.profiles), + plus_api_key=config.plus_api.is_active(), + ) diff --git a/frigate/test/test_analytics_collectors_features.py b/frigate/test/test_analytics_collectors_features.py new file mode 100644 index 0000000000..841ac76040 --- /dev/null +++ b/frigate/test/test_analytics_collectors_features.py @@ -0,0 +1,128 @@ +"""Tests for the features collector.""" + +import logging +import os +import unittest + +from peewee_migrate import Router +from playhouse.sqlite_ext import SqliteExtDatabase +from playhouse.sqliteq import SqliteQueueDatabase + +from frigate.analytics.collectors import features +from frigate.analytics.schema import EnrichmentDevice, SemanticSearchModel +from frigate.models import User +from frigate.test.analytics_helpers import FRONT_CAMERA, make_config, make_context +from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS + +CONFIG = { + "mqtt": {"host": "mqtt", "enabled": True}, + "genai": { + "local": { + "provider": "ollama", + "model": "llava", + "base_url": "http://ollama:11434", + "roles": ["descriptions", "embeddings"], + } + }, + "semantic_search": {"enabled": True, "model": "local"}, + "face_recognition": {"enabled": True, "model_size": "large"}, + "birdseye": {"enabled": True, "modes": ["motion", "alerts"], "restream": True}, + "proxy": {"header_map": {"user": "x-forwarded-user"}}, + "camera_groups": {"outside": {"cameras": ["front"], "icon": "LuCar", "order": 0}}, + "cameras": { + "front": { + **FRONT_CAMERA, + "semantic_search": { + "triggers": { + "cat": { + "type": "description", + "data": "a cat", + "threshold": 0.8, + "actions": ["notification"], + } + } + }, + } + }, +} + + +class TestFeaturesCollector(unittest.TestCase): + def setUp(self): + migrate_db = SqliteExtDatabase(TEST_DB) + del logging.getLogger("peewee_migrate").handlers[:] + Router(migrate_db).run() + migrate_db.close() + self.db = SqliteQueueDatabase(TEST_DB) + self.db.bind([User]) + + def tearDown(self): + # close() leaves the queue's writer thread running against the deleted file + self.db.stop() + + if not self.db.is_closed(): + self.db.close() + + for file in TEST_DB_CLEANUPS: + try: + os.remove(file) + except OSError: + pass + + def test_collects_enrichments_genai_integrations_and_users(self): + for username, role in (("a", "admin"), ("v", "viewer"), ("o", "operator")): + User.create( + username=username, role=role, password_hash="x", notification_tokens=[] + ) + + stats = { + "embeddings": { + "devices": { + "face_recognition": "OpenVINO GPU.0", + "semantic_search": "CUDA", + } + } + } + + section = features.collect(make_context(make_config(CONFIG), stats)) + + self.assertEqual(section.face_recognition.device, EnrichmentDevice.openvino_gpu) + self.assertEqual(section.face_recognition.model_size, "large") + self.assertIsNone(section.lpr.device) + self.assertEqual(section.semantic_search.model, SemanticSearchModel.genai) + self.assertEqual(section.semantic_search.device, EnrichmentDevice.cuda) + self.assertEqual(section.semantic_search.triggers, 1) + self.assertEqual(section.genai.providers, {"ollama": 1}) + self.assertEqual(section.genai.roles, {"descriptions": 1, "embeddings": 1}) + self.assertEqual(section.birdseye.modes, ["motion", "alerts"]) + self.assertTrue(section.mqtt) + self.assertTrue(section.proxy_auth) + self.assertEqual(section.users, {"admin": 1, "viewer": 1, "custom": 1}) + self.assertEqual((section.camera_groups, section.profiles), (1, 0)) + self.assertFalse(section.plus_api_key) + + +class TestFeatureMappings(unittest.TestCase): + def test_enrichment_device_labels(self): + cases = { + "CPU": EnrichmentDevice.cpu, + "MIGraphX": EnrichmentDevice.migraphx, + "OpenVINO NPU": EnrichmentDevice.openvino_npu, + "OpenVINO GPU.0,CPU": EnrichmentDevice.openvino_gpu, + "OpenVINO": EnrichmentDevice.other, + "CoreML": EnrichmentDevice.other, + } + + for label, device in cases.items(): + with self.subTest(label=label): + self.assertEqual(features.enrichment_device(label), device) + + self.assertIsNone(features.enrichment_device(None)) + + def test_models_named_after_a_provider_are_genai(self): + self.assertEqual(features.semantic_model("jinav2"), SemanticSearchModel.jinav2) + self.assertEqual( + features.semantic_model("my-ollama"), SemanticSearchModel.genai + ) + self.assertIsNone(features.semantic_model(None)) + self.assertEqual(features.transcription_model("openai"), "genai")