mirror of
https://github.com/blakeblackshear/frigate.git
synced 2026-11-15 01:53:02 +03:00
Implement opt-in analytics (#24440)
* Add analytics opt-in setting, prompt notice kind, and notice watermarks * Add the analytics report schema * Add the analytics state file * Add the install and hardware analytics collectors * Add the detection analytics collector * Add the cameras analytics collector * Add the features analytics collector * Add the health analytics collector * Build analytics reports from the section collectors * Add the analytics report transport * Send the daily analytics report from the main process * Add the analytics preview API * Generate the analytics schema and check it in CI * Record the image variant in each Dockerfile * Show the analytics preview in telemetry settings * Document anonymous analytics * change wording * fixes * update network requirements docs * adapt analytics to free-form model scenes, split notice counts, and peewee 4 * fixes * change log levels * docstrings * docstrings * honor file-only analytics opt-outs and skip pre-consent notices An opt-out saved through the YAML editor's "Save only" or a hand edit never reached the live config, so reports kept going out under the old install ID until a restart. The reporter now reads telemetry.analytics from the config file when a report is due, validated with TelemetryConfig so yes, on, and 1 still count as consent, and an opt-out there deletes the state file and skips the send. The first report under a new install ID also carried every notice since the counters started, including ones from before the opt-in, so a fresh state file now resets the notice watermarks before the report's snapshot. * take the notice baseline at opt-in instead of the first send The watermark reset ran at the first send attempt, so notices raised between the opt-in and that send, up to 45 minutes after a startup, were absorbed into the reset and never reported. The reporter now writes the new install ID and resets the watermarks as soon as it sees consent without a state file. * report notice counts cumulatively from a database baseline Moving watermarks after each accepted send kept the install ID and the notice baseline out of step: pre-consent notices went out, notices raised before the first send were dropped, and a send that finished after an off/on toggle overwrote the new identity's baseline. health.notices now carries cumulative counts since the install ID was created. Creating an ID copies the lifetime counts into the existing reported_* columns before the state file names it, and nothing is written after a send, so the backend diffs consecutive reports per install ID. * write held notice repeats before the analytics baseline reset Repeats of batched kinds like failed_login wait in memory until the stats emitter flushes them, so a baseline reset taken while some were held left them out, and the next flush counted those pre-opt-in repeats as post-consent. The reset now writes held repeats to the notice history under the same lock before copying the counts. * seed detector inference speed from the first inference The average started at 0 so idle detectors read 0, but that made every detector's speed climb from 0 for its first couple dozen inferences after a start. Detectors now use the same InferenceSpeed helper as enrichments, so the first inference seeds the average and idle detectors still read 0.
This commit is contained in:
Josh Hawkins
@@ -0,0 +1,36 @@
|
||||
"""Shared fixtures for the analytics tests."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from frigate.analytics.context import ReportContext
|
||||
from frigate.config import FrigateConfig
|
||||
|
||||
FRONT_CAMERA: dict[str, Any] = {
|
||||
"ffmpeg": {"inputs": [{"path": "rtsp://10.0.0.1:554/video", "roles": ["detect"]}]},
|
||||
"detect": {"height": 720, "width": 1280, "fps": 5},
|
||||
}
|
||||
|
||||
|
||||
def make_config(config: dict[str, Any] | None = None) -> FrigateConfig:
|
||||
"""A parsed config; top-level keys in config replace the defaults here.
|
||||
|
||||
hwaccel_args is set so parsing never probes the test host's GPU.
|
||||
"""
|
||||
base: dict[str, Any] = {
|
||||
"mqtt": {"host": "mqtt"},
|
||||
"ffmpeg": {"hwaccel_args": []},
|
||||
"cameras": {"front": FRONT_CAMERA},
|
||||
}
|
||||
return FrigateConfig(**{**base, **(config or {})})
|
||||
|
||||
|
||||
def make_context(
|
||||
config: FrigateConfig | None = None,
|
||||
stats: dict[str, Any] | None = None,
|
||||
notice_stats: list[dict[str, Any]] | None = None,
|
||||
) -> ReportContext:
|
||||
return ReportContext(
|
||||
config=config or make_config(),
|
||||
stats=stats or {},
|
||||
notice_stats=notice_stats or [],
|
||||
)
|
||||
@@ -0,0 +1,49 @@
|
||||
"""Tests for the analytics preview API."""
|
||||
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from frigate.analytics import report
|
||||
from frigate.analytics.collectors import hardware
|
||||
from frigate.models import Notice, NoticeStats, User
|
||||
from frigate.notices.registry import NoticeRegistry
|
||||
from frigate.test.http_api.base_http_test import AuthTestClient, BaseTestHttp
|
||||
|
||||
|
||||
class TestHttpAnalytics(BaseTestHttp):
|
||||
def setUp(self):
|
||||
super().setUp([Notice, NoticeStats, User])
|
||||
stats = Mock()
|
||||
stats.get_latest_stats.return_value = self.test_stats
|
||||
self.app = self.create_app(stats=stats, notice_registry=NoticeRegistry())
|
||||
|
||||
for target, name, value in (
|
||||
(report, "load_state", None),
|
||||
(hardware.hardware_prober, "probe", []),
|
||||
(hardware, "hwaccel_options", ("", [])),
|
||||
):
|
||||
patcher = patch.object(target, name, return_value=value)
|
||||
patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def test_preview_returns_the_report_without_sending_it(self):
|
||||
with (
|
||||
patch("frigate.analytics.transport.requests.post") as post,
|
||||
AuthTestClient(self.app) as client,
|
||||
):
|
||||
response = client.get("/analytics/preview")
|
||||
|
||||
self.assertEqual(response.status_code, 200)
|
||||
body = response.json()
|
||||
self.assertEqual(body["install_id"], report.PREVIEW_INSTALL_ID)
|
||||
self.assertEqual(body["schema_version"], 1)
|
||||
self.assertEqual(body["cameras"]["total"], 1)
|
||||
post.assert_not_called()
|
||||
|
||||
def test_preview_is_admin_only(self):
|
||||
with AuthTestClient(self.app) as client:
|
||||
response = client.get(
|
||||
"/analytics/preview",
|
||||
headers={"remote-user": "viewer", "remote-role": "viewer"},
|
||||
)
|
||||
|
||||
self.assertEqual(response.status_code, 403)
|
||||
@@ -0,0 +1,133 @@
|
||||
"""Tests for the cameras collector."""
|
||||
|
||||
import unittest
|
||||
|
||||
from frigate.analytics.collectors import cameras
|
||||
from frigate.analytics.schema import HeightBucket, HwaccelKey, RetainBucket
|
||||
from frigate.test.analytics_helpers import make_config, make_context
|
||||
|
||||
MASK = {"coordinates": "0,0,1,0,1,1"}
|
||||
|
||||
CONFIG = {
|
||||
"audio_transcription": {"enabled": True},
|
||||
"record": {
|
||||
"enabled": True,
|
||||
"continuous": {"days": 3},
|
||||
"alerts": {"retain": {"days": 14}},
|
||||
"detections": {"retain": {"days": 45}},
|
||||
},
|
||||
"cameras": {
|
||||
"front": {
|
||||
"ffmpeg": {
|
||||
"hwaccel_args": "preset-vaapi",
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://127.0.0.1:8554/front_sub",
|
||||
"roles": ["detect"],
|
||||
"input_args": "preset-rtsp-restream",
|
||||
},
|
||||
{"path": "rtsp://127.0.0.1:8554/front", "roles": ["record"]},
|
||||
],
|
||||
},
|
||||
"detect": {"enabled": True, "height": 720, "width": 1280, "fps": 5},
|
||||
"zones": {"porch": MASK, "yard": MASK},
|
||||
"motion": {"mask": {"tree": MASK}},
|
||||
"objects": {
|
||||
"mask": {"sign": MASK},
|
||||
"filters": {"person": {"mask": {"bush": MASK}}},
|
||||
},
|
||||
"onvif": {"host": "10.0.0.9"},
|
||||
},
|
||||
"garage": {
|
||||
"type": "lpr",
|
||||
"ffmpeg": {
|
||||
"hwaccel_args": ["-hwaccel", "vaapi"],
|
||||
"inputs": [
|
||||
{
|
||||
"path": "rtsp://10.0.0.5/stream",
|
||||
"roles": ["detect", "record", "audio"],
|
||||
}
|
||||
],
|
||||
},
|
||||
"audio": {"enabled": True},
|
||||
"detect": {"height": 2160, "width": 3840, "fps": 15},
|
||||
"record": {"enabled": False, "sub": {"enabled": True}},
|
||||
},
|
||||
"attic": {
|
||||
"enabled": False,
|
||||
"ffmpeg": {
|
||||
"inputs": [{"path": "rtsp://10.0.0.6/stream", "roles": ["detect"]}]
|
||||
},
|
||||
"detect": {"enabled": True, "height": 480, "width": 640, "fps": 5},
|
||||
},
|
||||
"_replay_front": {
|
||||
"ffmpeg": {
|
||||
"inputs": [{"path": "rtsp://10.0.0.1/replay", "roles": ["detect"]}]
|
||||
},
|
||||
"detect": {"height": 1080, "width": 1920, "fps": 5},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
STATS = {
|
||||
"cameras": {
|
||||
"front": {"connection_quality": "excellent"},
|
||||
"garage": {"connection_quality": "offline"},
|
||||
"attic": {"connection_quality": "unusable"},
|
||||
"_replay_front": {"connection_quality": "fair"},
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TestCamerasCollector(unittest.TestCase):
|
||||
def test_aggregates_enabled_cameras_and_skips_replays(self):
|
||||
section = cameras.collect(make_context(make_config(CONFIG), STATS))
|
||||
|
||||
self.assertEqual((section.total, section.enabled), (3, 2))
|
||||
self.assertEqual(section.types, {"generic": 1, "lpr": 1})
|
||||
self.assertEqual(section.detect_height, {"720": 1, "ge_2160": 1})
|
||||
self.assertEqual(section.detect_fps, {"le_5": 1, "gt_10": 1})
|
||||
self.assertEqual(section.hwaccel, {"vaapi": 1, "custom": 1})
|
||||
self.assertEqual(section.input_preset, {"rtsp-restream": 1, "rtsp-generic": 1})
|
||||
self.assertEqual(section.go2rtc_restream, 1)
|
||||
self.assertEqual(section.separate_detect_stream, 1)
|
||||
self.assertEqual((section.detect, section.record, section.onvif), (1, 1, 1))
|
||||
self.assertEqual((section.zones, section.cameras_with_zones), (2, 1))
|
||||
self.assertEqual((section.motion_masks, section.object_masks), (1, 2))
|
||||
self.assertEqual((section.audio, section.audio_transcription), (1, 1))
|
||||
self.assertEqual(section.sub_stream_record, 0)
|
||||
self.assertEqual(section.connection_quality, {"excellent": 1})
|
||||
|
||||
def test_runtime_disabled_camera_skips_connection_quality(self):
|
||||
config = make_config(CONFIG)
|
||||
config.cameras["front"].enabled = False
|
||||
section = cameras.collect(make_context(config, STATS))
|
||||
|
||||
self.assertEqual(section.connection_quality, {})
|
||||
self.assertEqual(section.retain_days.continuous, {"1_7": 1})
|
||||
self.assertEqual(section.retain_days.motion, {"0": 1})
|
||||
self.assertEqual(section.retain_days.alerts, {"8_30": 1})
|
||||
self.assertEqual(section.retain_days.detections, {"gt_30": 1})
|
||||
|
||||
def test_buckets(self):
|
||||
self.assertEqual(cameras.height_bucket(360), HeightBucket.le_360)
|
||||
self.assertEqual(cameras.height_bucket(361), HeightBucket.h480)
|
||||
self.assertEqual(cameras.height_bucket(1080), HeightBucket.h1080)
|
||||
self.assertEqual(cameras.height_bucket(1440), HeightBucket.h1440)
|
||||
self.assertEqual(cameras.height_bucket(2160), HeightBucket.ge_2160)
|
||||
self.assertEqual(cameras.retain_bucket(0.5), RetainBucket.d1_7)
|
||||
self.assertEqual(cameras.retain_bucket(0), RetainBucket.zero)
|
||||
|
||||
def test_preset_key(self):
|
||||
self.assertEqual(cameras.preset_key([], HwaccelKey), "none")
|
||||
self.assertEqual(cameras.preset_key("", HwaccelKey), "none")
|
||||
self.assertEqual(cameras.preset_key("preset-bogus", HwaccelKey), "custom")
|
||||
self.assertEqual(cameras.preset_key("-hwaccel vaapi", HwaccelKey), "custom")
|
||||
self.assertEqual(cameras.preset_key("preset-nvidia", HwaccelKey), "nvidia")
|
||||
|
||||
def test_is_restream(self):
|
||||
self.assertTrue(cameras.is_restream("rtsp://127.0.0.1:8554/front"))
|
||||
self.assertTrue(cameras.is_restream("rtsp://localhost:8554/front"))
|
||||
self.assertFalse(cameras.is_restream("rtsp://10.0.0.5:8554/front"))
|
||||
self.assertFalse(cameras.is_restream("rtsp://[bad"))
|
||||
self.assertFalse(cameras.is_restream("rtsp://127.0.0.1:notaport/x"))
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Tests for the detection collector."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from frigate.analytics.collectors import detection
|
||||
from frigate.analytics.schema import ModelSource
|
||||
from frigate.test.analytics_helpers import make_config, make_context
|
||||
|
||||
|
||||
class TestDetectionCollector(unittest.TestCase):
|
||||
def test_reports_each_model_with_its_mean_inference_speed(self):
|
||||
config = make_config({"models": [{"devices": ["cpu", "cpu"]}]})
|
||||
stats = {
|
||||
"detectors": {
|
||||
"cpu": {"inference_speed": 10.0},
|
||||
"cpu#2": {"inference_speed": 20.0},
|
||||
},
|
||||
"detection_fps": 12.346,
|
||||
"skipped_fps": 0,
|
||||
}
|
||||
|
||||
section = detection.collect(make_context(config, stats))
|
||||
|
||||
[model] = section.models
|
||||
self.assertTrue(model.default)
|
||||
self.assertEqual(model.detector, "cpu")
|
||||
self.assertEqual(model.devices, 2)
|
||||
self.assertEqual(model.model_type, "ssd")
|
||||
self.assertEqual(model.input, "320x320")
|
||||
self.assertEqual(model.source, ModelSource.default)
|
||||
self.assertEqual(model.inference_ms, 15.0)
|
||||
self.assertEqual(section.detection_fps, 12.35)
|
||||
self.assertEqual(section.skipped_fps, 0.0)
|
||||
|
||||
def test_idle_detector_reports_no_speed(self):
|
||||
config = make_config({"models": [{"devices": ["cpu"]}]})
|
||||
stats = {"detectors": {"cpu": {"inference_speed": 0.0}}}
|
||||
|
||||
[model] = detection.collect(make_context(config, stats)).models
|
||||
self.assertIsNone(model.inference_ms)
|
||||
|
||||
def test_named_scene_reports_no_name(self):
|
||||
config = make_config(
|
||||
{
|
||||
"models": [
|
||||
{"devices": ["cpu"]},
|
||||
{
|
||||
"scene": "backyard_thermal",
|
||||
"devices": ["cpu"],
|
||||
"path": "/thermal.tflite",
|
||||
},
|
||||
]
|
||||
}
|
||||
)
|
||||
|
||||
section = detection.collect(make_context(config))
|
||||
|
||||
self.assertEqual([m.default for m in section.models], [True, False])
|
||||
self.assertNotIn("backyard_thermal", section.model_dump_json())
|
||||
|
||||
def test_inference_is_null_before_the_first_stats(self):
|
||||
section = detection.collect(make_context())
|
||||
|
||||
self.assertIsNone(section.models[0].inference_ms)
|
||||
|
||||
def test_model_source(self):
|
||||
with tempfile.TemporaryDirectory() as cache:
|
||||
plus_model = os.path.join(cache, "abc123")
|
||||
open(f"{plus_model}.json", "w").close()
|
||||
|
||||
with patch.object(detection, "MODEL_CACHE_DIR", cache):
|
||||
self.assertEqual(detection.model_source(plus_model), ModelSource.plus)
|
||||
self.assertEqual(
|
||||
detection.model_source(os.path.join(cache, "no_info")),
|
||||
ModelSource.custom,
|
||||
)
|
||||
|
||||
self.assertEqual(detection.model_source(None), ModelSource.default)
|
||||
self.assertEqual(
|
||||
detection.model_source("/cpu_model.tflite"), ModelSource.default
|
||||
)
|
||||
self.assertEqual(
|
||||
detection.model_source("/config/yolo.onnx"), ModelSource.custom
|
||||
)
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Tests for the features collector."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import unittest
|
||||
|
||||
from peewee import SqliteDatabase
|
||||
from peewee_migrate import Router
|
||||
from playhouse.sqliteq import SqliteQueueDatabase
|
||||
|
||||
from frigate.analytics.collectors import features
|
||||
from frigate.analytics.schema import EnrichmentDevice, SemanticSearchModel
|
||||
from frigate.models import User
|
||||
from frigate.test.analytics_helpers import FRONT_CAMERA, make_config, make_context
|
||||
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
|
||||
|
||||
CONFIG = {
|
||||
"mqtt": {"host": "mqtt", "enabled": True},
|
||||
"genai": {
|
||||
"local": {
|
||||
"provider": "ollama",
|
||||
"model": "llava",
|
||||
"base_url": "http://ollama:11434",
|
||||
"roles": ["descriptions", "embeddings"],
|
||||
}
|
||||
},
|
||||
"semantic_search": {"enabled": True, "model": "local"},
|
||||
"face_recognition": {"enabled": True, "model_size": "large"},
|
||||
"birdseye": {"enabled": True, "modes": ["motion", "alerts"], "restream": True},
|
||||
"proxy": {"header_map": {"user": "x-forwarded-user"}},
|
||||
"camera_groups": {"outside": {"cameras": ["front"], "icon": "LuCar", "order": 0}},
|
||||
"cameras": {
|
||||
"front": {
|
||||
**FRONT_CAMERA,
|
||||
"semantic_search": {
|
||||
"triggers": {
|
||||
"cat": {
|
||||
"type": "description",
|
||||
"data": "a cat",
|
||||
"threshold": 0.8,
|
||||
"actions": ["notification"],
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class TestFeaturesCollector(unittest.TestCase):
|
||||
def setUp(self):
|
||||
migrate_db = SqliteDatabase(TEST_DB)
|
||||
del logging.getLogger("peewee_migrate").handlers[:]
|
||||
Router(migrate_db).run()
|
||||
migrate_db.close()
|
||||
self.db = SqliteQueueDatabase(TEST_DB)
|
||||
self.db.bind([User])
|
||||
|
||||
def tearDown(self):
|
||||
# close() leaves the queue's writer thread running against the deleted file
|
||||
self.db.stop()
|
||||
|
||||
if not self.db.is_closed():
|
||||
self.db.close()
|
||||
|
||||
for file in TEST_DB_CLEANUPS:
|
||||
try:
|
||||
os.remove(file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_collects_enrichments_genai_integrations_and_users(self):
|
||||
for username, role in (("a", "admin"), ("v", "viewer"), ("o", "operator")):
|
||||
User.create(
|
||||
username=username, role=role, password_hash="x", notification_tokens=[]
|
||||
)
|
||||
|
||||
stats = {
|
||||
"embeddings": {
|
||||
"devices": {
|
||||
"face_recognition": "OpenVINO GPU.0",
|
||||
"semantic_search": "CUDA",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
section = features.collect(make_context(make_config(CONFIG), stats))
|
||||
|
||||
self.assertEqual(section.face_recognition.device, EnrichmentDevice.openvino_gpu)
|
||||
self.assertEqual(section.face_recognition.model_size, "large")
|
||||
self.assertIsNone(section.lpr.device)
|
||||
self.assertEqual(section.semantic_search.model, SemanticSearchModel.genai)
|
||||
self.assertEqual(section.semantic_search.device, EnrichmentDevice.cuda)
|
||||
self.assertEqual(section.semantic_search.triggers, 1)
|
||||
self.assertEqual(section.genai.providers, {"ollama": 1})
|
||||
self.assertEqual(section.genai.roles, {"descriptions": 1, "embeddings": 1})
|
||||
self.assertEqual(section.birdseye.modes, ["motion", "alerts"])
|
||||
self.assertTrue(section.mqtt)
|
||||
self.assertTrue(section.proxy_auth)
|
||||
self.assertEqual(section.users, {"admin": 1, "viewer": 1, "custom": 1})
|
||||
self.assertEqual((section.camera_groups, section.profiles), (1, 0))
|
||||
self.assertFalse(section.plus_api_key)
|
||||
|
||||
|
||||
class TestFeatureMappings(unittest.TestCase):
|
||||
def test_enrichment_device_labels(self):
|
||||
cases = {
|
||||
"CPU": EnrichmentDevice.cpu,
|
||||
"MIGraphX": EnrichmentDevice.migraphx,
|
||||
"OpenVINO NPU": EnrichmentDevice.openvino_npu,
|
||||
"OpenVINO GPU.0,CPU": EnrichmentDevice.openvino_gpu,
|
||||
"OpenVINO (CPU)": EnrichmentDevice.openvino_cpu,
|
||||
"OpenVINO": EnrichmentDevice.other,
|
||||
"CoreML": EnrichmentDevice.other,
|
||||
}
|
||||
|
||||
for label, device in cases.items():
|
||||
with self.subTest(label=label):
|
||||
self.assertEqual(features.enrichment_device(label), device)
|
||||
|
||||
self.assertIsNone(features.enrichment_device(None))
|
||||
|
||||
def test_models_named_after_a_provider_are_genai(self):
|
||||
self.assertEqual(features.semantic_model("jinav2"), SemanticSearchModel.jinav2)
|
||||
self.assertEqual(
|
||||
features.semantic_model("my-ollama"), SemanticSearchModel.genai
|
||||
)
|
||||
self.assertIsNone(features.semantic_model(None))
|
||||
self.assertEqual(features.transcription_model("openai"), "genai")
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Tests for the health collector."""
|
||||
|
||||
import unittest
|
||||
|
||||
from frigate.analytics.collectors import health
|
||||
from frigate.test.analytics_helpers import make_context
|
||||
|
||||
|
||||
class TestHealthCollector(unittest.TestCase):
|
||||
def test_collects_uptime_cpu_timings_and_notices_since_the_baseline(self):
|
||||
stats = {
|
||||
"service": {"uptime": 7300, "retention_unmet": True},
|
||||
"cpu_usages": {"frigate.full_system": {"cpu": "23.6", "mem": "40"}},
|
||||
"embeddings": {
|
||||
"face_recognition_speed": 41.234,
|
||||
"text_embedding_speed": 0,
|
||||
"devices": {},
|
||||
},
|
||||
}
|
||||
notice_stats = [
|
||||
{
|
||||
"kind": "detector_stuck",
|
||||
"occurrences": 5,
|
||||
"acknowledgements": 1,
|
||||
"mutes": 0,
|
||||
"reported_occurrences": 3,
|
||||
"reported_acknowledgements": 1,
|
||||
"reported_mutes": 0,
|
||||
},
|
||||
{
|
||||
"kind": "shm_too_low",
|
||||
"occurrences": 2,
|
||||
"acknowledgements": 0,
|
||||
"mutes": 0,
|
||||
"reported_occurrences": 2,
|
||||
"reported_acknowledgements": 0,
|
||||
"reported_mutes": 0,
|
||||
},
|
||||
{
|
||||
"kind": "update_available",
|
||||
"occurrences": 4,
|
||||
"acknowledgements": 0,
|
||||
"mutes": 0,
|
||||
"reported_occurrences": 0,
|
||||
"reported_acknowledgements": 0,
|
||||
"reported_mutes": 0,
|
||||
},
|
||||
]
|
||||
|
||||
section = health.collect(make_context(stats=stats, notice_stats=notice_stats))
|
||||
|
||||
self.assertEqual(section.uptime_hours, 2)
|
||||
self.assertEqual(section.cpu_percent, 24)
|
||||
self.assertEqual(section.enrichment_ms, {"face": 41.23})
|
||||
self.assertTrue(section.retention_unmet)
|
||||
self.assertEqual(
|
||||
{
|
||||
kind: (c.occurrences, c.acknowledgements, c.mutes)
|
||||
for kind, c in section.notices.items()
|
||||
},
|
||||
{"detector_stuck": (2, 0, 0)},
|
||||
)
|
||||
|
||||
def test_missing_stats_give_zeros(self):
|
||||
section = health.collect(make_context())
|
||||
|
||||
self.assertEqual(
|
||||
(section.uptime_hours, section.cpu_percent, section.enrichment_ms),
|
||||
(0, 0, {}),
|
||||
)
|
||||
self.assertEqual(section.notices, {})
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Tests for the install and hardware collectors."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from collections import Counter
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from frigate.analytics.collectors import hardware, install
|
||||
from frigate.analytics.collectors.common import closed, histogram, rate
|
||||
from frigate.analytics.schema import (
|
||||
Arch,
|
||||
DecodeFamily,
|
||||
GpuVendor,
|
||||
HardwareKey,
|
||||
ImageVariant,
|
||||
InstallType,
|
||||
)
|
||||
from frigate.const import RECORD_DIR
|
||||
from frigate.test.analytics_helpers import make_context
|
||||
|
||||
|
||||
class TestCommon(unittest.TestCase):
|
||||
def test_closed_falls_back_for_unknown_values(self):
|
||||
self.assertEqual(closed(Arch, "x86_64", Arch.other), Arch.x86_64)
|
||||
self.assertEqual(closed(Arch, "sparc", Arch.other), Arch.other)
|
||||
self.assertEqual(closed(Arch, None, Arch.other), Arch.other)
|
||||
|
||||
def test_rate_rounds_and_rejects_bad_numbers(self):
|
||||
self.assertEqual(rate("12.346"), 12.35)
|
||||
self.assertEqual(rate(-3), 0.0)
|
||||
self.assertEqual(rate(float("nan")), 0.0)
|
||||
self.assertEqual(rate(float("inf")), 0.0)
|
||||
self.assertEqual(rate(None), 0.0)
|
||||
|
||||
def test_histogram_drops_empty_buckets(self):
|
||||
self.assertEqual(
|
||||
histogram(Counter({Arch.x86_64: 2, Arch.other: 0})), {"x86_64": 2}
|
||||
)
|
||||
|
||||
|
||||
class TestInstallCollector(unittest.TestCase):
|
||||
def test_collects_version_variant_and_platform(self):
|
||||
with (
|
||||
patch.dict(os.environ, {"FRIGATE_IMAGE_VARIANT": "tensorrt-jp6"}),
|
||||
patch.object(install, "install_type", return_value=InstallType.docker),
|
||||
patch("platform.machine", return_value="aarch64"),
|
||||
patch("platform.release", return_value="6.8.0-45-generic"),
|
||||
patch("os.geteuid", return_value=1000),
|
||||
):
|
||||
section = install.collect(make_context())
|
||||
|
||||
self.assertEqual(section.image_variant, ImageVariant.tensorrt_jp6)
|
||||
self.assertEqual(section.install_type, InstallType.docker)
|
||||
self.assertEqual(section.arch, Arch.aarch64)
|
||||
self.assertEqual(section.kernel, "6.8")
|
||||
self.assertFalse(section.run_as_root)
|
||||
|
||||
def test_image_variant_is_dev_when_unset_and_other_when_unknown(self):
|
||||
self.assertEqual(install.image_variant(None), ImageVariant.dev)
|
||||
self.assertEqual(install.image_variant(""), ImageVariant.dev)
|
||||
self.assertEqual(install.image_variant("h8l"), ImageVariant.other)
|
||||
|
||||
def test_install_type_checks_the_add_on_first(self):
|
||||
with patch("os.path.isfile", return_value=True):
|
||||
self.assertEqual(install.install_type(), InstallType.ha_addon)
|
||||
|
||||
with (
|
||||
patch("os.path.isfile", return_value=False),
|
||||
patch.dict(os.environ, {"KUBERNETES_SERVICE_HOST": "10.0.0.1"}),
|
||||
):
|
||||
self.assertEqual(install.install_type(), InstallType.kubernetes)
|
||||
|
||||
env = {k: v for k, v in os.environ.items() if k != "KUBERNETES_SERVICE_HOST"}
|
||||
|
||||
with (
|
||||
patch("os.path.isfile", return_value=False),
|
||||
patch.dict(os.environ, env, clear=True),
|
||||
patch("os.path.exists", side_effect=lambda path: path == "/.dockerenv"),
|
||||
):
|
||||
self.assertEqual(install.install_type(), InstallType.docker)
|
||||
|
||||
def test_kernel_and_arch_fall_back(self):
|
||||
self.assertEqual(install.kernel("weird"), "unknown")
|
||||
self.assertEqual(install.arch("armv7l"), Arch.other)
|
||||
self.assertEqual(install.arch("amd64"), Arch.x86_64)
|
||||
|
||||
|
||||
class TestHardwareCollector(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.dir.cleanup)
|
||||
|
||||
def write(self, name: str, content: str) -> str:
|
||||
path = os.path.join(self.dir.name, name)
|
||||
|
||||
with open(path, "w") as f:
|
||||
f.write(content)
|
||||
|
||||
return path
|
||||
|
||||
def test_cpu_model_prefers_the_board_model(self):
|
||||
board = self.write("model", "Raspberry Pi 5 Model B Rev 1.0\x00")
|
||||
cpuinfo = self.write("cpuinfo", "model name\t: Cortex-A76\n")
|
||||
|
||||
with (
|
||||
patch.object(hardware, "DEVICE_TREE_MODEL", board),
|
||||
patch.object(hardware, "CPUINFO", cpuinfo),
|
||||
):
|
||||
self.assertEqual(hardware.cpu_model(), "Raspberry Pi 5 Model B Rev 1.0")
|
||||
|
||||
def test_cpu_model_reads_cpuinfo_and_truncates(self):
|
||||
cpuinfo = self.write("cpuinfo", "processor\t: 0\nmodel name\t: " + "x" * 80)
|
||||
|
||||
with (
|
||||
patch.object(hardware, "DEVICE_TREE_MODEL", self.dir.name + "/none"),
|
||||
patch.object(hardware, "CPUINFO", cpuinfo),
|
||||
):
|
||||
self.assertEqual(hardware.cpu_model(), "x" * 64)
|
||||
|
||||
def test_collects_gpus_decode_detection_and_storage(self):
|
||||
stats = {
|
||||
"gpu_usages": {
|
||||
"NVIDIA GeForce RTX 3060": {"vendor": "nvidia"},
|
||||
"mystery": {},
|
||||
},
|
||||
"service": {
|
||||
"storage": {
|
||||
RECORD_DIR: {
|
||||
"total": 2048000.0,
|
||||
"used": 512000.0,
|
||||
"mount_type": "nfs4",
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
probed = [
|
||||
SimpleNamespace(key="edgetpu:usb", count=2),
|
||||
SimpleNamespace(key="new:thing", count=1),
|
||||
SimpleNamespace(key="cpu", count=1),
|
||||
]
|
||||
families = [
|
||||
SimpleNamespace(key="vaapi"),
|
||||
SimpleNamespace(key="intel-qsv"),
|
||||
SimpleNamespace(key="future"),
|
||||
]
|
||||
|
||||
with (
|
||||
patch.object(hardware.hardware_prober, "probe", return_value=probed),
|
||||
patch.object(hardware, "hwaccel_options", return_value=("vaapi", families)),
|
||||
patch.object(hardware, "cpu_model", return_value="Intel(R) N100"),
|
||||
patch("os.cpu_count", return_value=4),
|
||||
patch(
|
||||
"psutil.virtual_memory", return_value=SimpleNamespace(total=16 * 2**30)
|
||||
),
|
||||
):
|
||||
section = hardware.collect(make_context(stats=stats))
|
||||
|
||||
self.assertEqual(section.cpu_model, "Intel(R) N100")
|
||||
self.assertEqual(section.cpu_cores, 4)
|
||||
self.assertEqual(section.memory_gb, 16)
|
||||
self.assertEqual(
|
||||
[(gpu.vendor, gpu.name) for gpu in section.gpus],
|
||||
[
|
||||
(GpuVendor.nvidia, "NVIDIA GeForce RTX 3060"),
|
||||
(GpuVendor.other, "mystery"),
|
||||
],
|
||||
)
|
||||
self.assertEqual(
|
||||
section.decode_families,
|
||||
[DecodeFamily.vaapi, DecodeFamily.intel_qsv, DecodeFamily.other],
|
||||
)
|
||||
self.assertEqual(
|
||||
section.detection_hardware,
|
||||
{HardwareKey.edgetpu_usb: 2, HardwareKey.other: 1, HardwareKey.cpu: 1},
|
||||
)
|
||||
self.assertEqual(section.storage.record_fs, "nfs4")
|
||||
self.assertEqual(section.storage.record_total_gb, 2000)
|
||||
self.assertEqual(section.storage.record_used_pct, 25)
|
||||
|
||||
def test_storage_without_stats_is_unknown_and_empty(self):
|
||||
section = hardware.storage({})
|
||||
|
||||
self.assertEqual(
|
||||
(section.record_fs, section.record_total_gb, section.record_used_pct),
|
||||
("unknown", 0, 0),
|
||||
)
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Tests for building a report from the collectors."""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from peewee import SqliteDatabase
|
||||
from peewee_migrate import Router
|
||||
from playhouse.sqliteq import SqliteQueueDatabase
|
||||
|
||||
from frigate.analytics import report
|
||||
from frigate.analytics.collectors import hardware
|
||||
from frigate.analytics.schema import AnalyticsReport, InstallSection
|
||||
from frigate.analytics.state import AnalyticsState
|
||||
from frigate.models import User
|
||||
from frigate.test.analytics_helpers import make_context
|
||||
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
|
||||
|
||||
INSTALL = InstallSection(
|
||||
version="0.19.0",
|
||||
image_variant="dev",
|
||||
install_type="docker",
|
||||
arch="x86_64",
|
||||
kernel="6.8",
|
||||
run_as_root=False,
|
||||
)
|
||||
|
||||
|
||||
class TestBuildReport(unittest.TestCase):
|
||||
def test_a_failing_collector_nulls_only_its_section(self):
|
||||
collectors = {
|
||||
"install": lambda ctx: INSTALL,
|
||||
"hardware": Mock(side_effect=RuntimeError("boom")),
|
||||
"detection": lambda ctx: None,
|
||||
"cameras": lambda ctx: None,
|
||||
"features": lambda ctx: None,
|
||||
"health": lambda ctx: None,
|
||||
}
|
||||
|
||||
with patch.dict(report.COLLECTORS, collectors):
|
||||
built = report.build_report(make_context(), "a" * 32, sent_at=1790000000)
|
||||
|
||||
self.assertEqual(built.install, INSTALL)
|
||||
self.assertIsNone(built.hardware)
|
||||
self.assertEqual(built.schema_version, 1)
|
||||
self.assertEqual(built.sent_at, 1790000000)
|
||||
self.assertRegex(built.report_id, r"^[0-9a-f-]{36}$")
|
||||
|
||||
|
||||
class TestRealCollectors(unittest.TestCase):
|
||||
def setUp(self):
|
||||
migrate_db = SqliteDatabase(TEST_DB)
|
||||
del logging.getLogger("peewee_migrate").handlers[:]
|
||||
Router(migrate_db).run()
|
||||
migrate_db.close()
|
||||
self.db = SqliteQueueDatabase(TEST_DB)
|
||||
self.db.bind([User])
|
||||
|
||||
def tearDown(self):
|
||||
# close() leaves the queue's writer thread running against the deleted file
|
||||
self.db.stop()
|
||||
|
||||
if not self.db.is_closed():
|
||||
self.db.close()
|
||||
|
||||
for file in TEST_DB_CLEANUPS:
|
||||
try:
|
||||
os.remove(file)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def test_every_section_builds_and_round_trips_through_json(self):
|
||||
with (
|
||||
patch.object(hardware.hardware_prober, "probe", return_value=[]),
|
||||
patch.object(hardware, "hwaccel_options", return_value=("", [])),
|
||||
):
|
||||
built = report.build_report(make_context(), "a" * 32)
|
||||
|
||||
body = built.model_dump_json()
|
||||
parsed = json.loads(body)
|
||||
|
||||
for section in (
|
||||
"install",
|
||||
"hardware",
|
||||
"detection",
|
||||
"cameras",
|
||||
"features",
|
||||
"health",
|
||||
):
|
||||
with self.subTest(section=section):
|
||||
self.assertIsNotNone(parsed[section])
|
||||
|
||||
self.assertEqual(AnalyticsReport.model_validate_json(body), built)
|
||||
|
||||
def test_preview_uses_a_placeholder_id_before_the_first_report(self):
|
||||
stats_emitter = Mock()
|
||||
stats_emitter.get_latest_stats.return_value = {}
|
||||
|
||||
with (
|
||||
patch.object(report, "load_state", return_value=None),
|
||||
patch.object(hardware.hardware_prober, "probe", return_value=[]),
|
||||
patch.object(hardware, "hwaccel_options", return_value=("", [])),
|
||||
):
|
||||
built = report.preview_report(make_context().config, stats_emitter, None)
|
||||
|
||||
self.assertEqual(built.install_id, report.PREVIEW_INSTALL_ID)
|
||||
|
||||
def test_preview_counts_notices_only_once_opted_in(self):
|
||||
stats_emitter = Mock()
|
||||
stats_emitter.get_latest_stats.return_value = {}
|
||||
registry = Mock()
|
||||
registry.stats.return_value = [
|
||||
{
|
||||
"kind": "detector_stuck",
|
||||
"occurrences": 3,
|
||||
"acknowledgements": 0,
|
||||
"mutes": 0,
|
||||
"reported_occurrences": 1,
|
||||
"reported_acknowledgements": 0,
|
||||
"reported_mutes": 0,
|
||||
}
|
||||
]
|
||||
|
||||
for state, expected in (
|
||||
(None, {}),
|
||||
(AnalyticsState("a" * 32, 0.0), {"detector_stuck": 2}),
|
||||
):
|
||||
with (
|
||||
self.subTest(opted_in=state is not None),
|
||||
patch.object(report, "load_state", return_value=state),
|
||||
patch.object(hardware.hardware_prober, "probe", return_value=[]),
|
||||
patch.object(hardware, "hwaccel_options", return_value=("", [])),
|
||||
):
|
||||
built = report.preview_report(
|
||||
make_context().config, stats_emitter, registry
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
{kind: c.occurrences for kind, c in built.health.notices.items()},
|
||||
expected,
|
||||
)
|
||||
@@ -0,0 +1,349 @@
|
||||
"""Tests for the analytics reporter's schedule and consent handling."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from frigate.analytics import reporter as reporter_module
|
||||
from frigate.analytics.reporter import (
|
||||
FIRST_DELAY_S,
|
||||
INTERVAL_S,
|
||||
JITTER_S,
|
||||
PROMPT_KIND,
|
||||
WAKE_S,
|
||||
AnalyticsReporter,
|
||||
)
|
||||
from frigate.analytics.state import AnalyticsState, load_state, save_state
|
||||
from frigate.analytics.transport import SendOutcome
|
||||
from frigate.config.holder import ConfigHolder
|
||||
from frigate.test.analytics_helpers import make_config
|
||||
|
||||
|
||||
class TestAnalyticsReporter(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.dir.cleanup)
|
||||
self.path = os.path.join(self.dir.name, ".analytics.json")
|
||||
self.config_path = os.path.join(self.dir.name, "config.yml")
|
||||
self.save_config("telemetry:\n analytics: true\n")
|
||||
self.now = 1_790_000_000.0
|
||||
self.send = Mock(return_value=SendOutcome.accepted)
|
||||
self.registry = Mock()
|
||||
self.stats = Mock()
|
||||
self.stats.get_latest_stats.return_value = {}
|
||||
self.holder = ConfigHolder(make_config({"telemetry": {"analytics": True}}))
|
||||
|
||||
for name in ("raise_notice", "resolve_notice"):
|
||||
patcher = patch.object(reporter_module, name)
|
||||
setattr(self, name, patcher.start())
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
self.built = Mock()
|
||||
self.built.model_dump_json.return_value = '{"report": 1}'
|
||||
patcher = patch.object(reporter_module, "build_report", return_value=self.built)
|
||||
self.build_report = patcher.start()
|
||||
self.addCleanup(patcher.stop)
|
||||
|
||||
def save_config(self, text: str) -> None:
|
||||
with open(self.config_path, "w") as f:
|
||||
f.write(text)
|
||||
|
||||
def seed_state(self, last_attempt_at: float = 0.0) -> None:
|
||||
save_state(AnalyticsState("a" * 32, last_attempt_at), self.path)
|
||||
|
||||
def reporter(self, path: str | None = None) -> AnalyticsReporter:
|
||||
# uniform(a, b) returns a: the shortest startup delay and interval
|
||||
rng = Mock()
|
||||
rng.uniform.side_effect = lambda low, high: low
|
||||
|
||||
return AnalyticsReporter(
|
||||
self.holder,
|
||||
self.stats,
|
||||
self.registry,
|
||||
threading.Event(),
|
||||
state_path=path or self.path,
|
||||
config_path=self.config_path,
|
||||
url="https://example.test/report",
|
||||
send=self.send,
|
||||
clock=lambda: self.now,
|
||||
rng=rng,
|
||||
)
|
||||
|
||||
def advance(self, seconds: float) -> None:
|
||||
self.now += seconds
|
||||
|
||||
def test_waits_for_the_startup_delay_then_sends(self):
|
||||
self.seed_state()
|
||||
reporter = self.reporter()
|
||||
|
||||
reporter.tick()
|
||||
self.send.assert_not_called()
|
||||
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once_with(
|
||||
"https://example.test/report", '{"report": 1}'
|
||||
)
|
||||
self.assertEqual(load_state(self.path).last_attempt_at, self.now)
|
||||
|
||||
def test_sends_once_per_interval(self):
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.advance(WAKE_S)
|
||||
reporter.tick()
|
||||
self.assertEqual(self.send.call_count, 1)
|
||||
|
||||
self.advance(INTERVAL_S - JITTER_S)
|
||||
reporter.tick()
|
||||
self.assertEqual(self.send.call_count, 2)
|
||||
|
||||
def test_a_restart_inside_the_interval_does_not_send(self):
|
||||
self.seed_state(self.now)
|
||||
reporter = self.reporter()
|
||||
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_not_called()
|
||||
|
||||
def test_a_failed_send_waits_a_full_interval(self):
|
||||
self.send.return_value = SendOutcome.failed
|
||||
self.seed_state()
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
reporter.tick()
|
||||
self.advance(WAKE_S)
|
||||
reporter.tick()
|
||||
|
||||
self.assertEqual(self.send.call_count, 1)
|
||||
|
||||
def assert_reset_before_the_state_file(self) -> None:
|
||||
# the baseline must be in place before the file names the new ID
|
||||
self.registry.reset_analytics_baseline.side_effect = lambda: self.assertIsNone(
|
||||
load_state(self.path)
|
||||
)
|
||||
|
||||
def test_the_opt_in_resets_the_baseline_and_writes_the_identity(self):
|
||||
self.assert_reset_before_the_state_file()
|
||||
reporter = self.reporter()
|
||||
|
||||
# consent is seen on the first wake, well before the first send
|
||||
reporter.tick()
|
||||
self.registry.reset_analytics_baseline.assert_called_once()
|
||||
self.assertIsNotNone(load_state(self.path))
|
||||
self.send.assert_not_called()
|
||||
|
||||
# the first report counts from that reset, so nothing resets again
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once()
|
||||
self.registry.reset_analytics_baseline.assert_called_once()
|
||||
|
||||
def test_a_restart_with_saved_state_keeps_the_baseline(self):
|
||||
self.seed_state()
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once()
|
||||
self.registry.reset_analytics_baseline.assert_not_called()
|
||||
|
||||
def test_a_settings_opt_in_resets_the_baseline_when_saved(self):
|
||||
self.holder.set(make_config())
|
||||
reporter = self.reporter()
|
||||
reporter.tick()
|
||||
|
||||
self.holder.set(make_config({"telemetry": {"analytics": True}}))
|
||||
|
||||
self.registry.reset_analytics_baseline.assert_called_once()
|
||||
self.assertIsNotNone(load_state(self.path))
|
||||
|
||||
def test_a_re_opt_in_during_the_request_keeps_its_new_identity(self):
|
||||
self.seed_state()
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
created = []
|
||||
|
||||
def flip_consent(*args):
|
||||
self.holder.set(make_config())
|
||||
self.holder.set(make_config({"telemetry": {"analytics": True}}))
|
||||
created.append(load_state(self.path))
|
||||
return SendOutcome.accepted
|
||||
|
||||
self.send.side_effect = flip_consent
|
||||
reporter.tick()
|
||||
|
||||
# nothing after the send touches the new identity or its baseline
|
||||
self.assertNotEqual(created[0].install_id, "a" * 32)
|
||||
self.assertEqual(load_state(self.path), created[0])
|
||||
self.registry.reset_analytics_baseline.assert_called_once()
|
||||
|
||||
def test_an_opt_in_without_a_writable_state_file_resets_again_later(self):
|
||||
state_dir = os.path.join(self.dir.name, "missing")
|
||||
reporter = self.reporter(os.path.join(state_dir, ".analytics.json"))
|
||||
|
||||
with self.assertLogs("frigate.analytics.reporter", "WARNING"):
|
||||
reporter.tick()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_not_called()
|
||||
os.mkdir(state_dir)
|
||||
self.registry.reset_analytics_baseline.reset_mock()
|
||||
self.advance(WAKE_S)
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once()
|
||||
self.registry.reset_analytics_baseline.assert_called_once()
|
||||
|
||||
def test_an_opt_out_saved_only_to_the_file_stops_the_send(self):
|
||||
self.seed_state()
|
||||
self.save_config("telemetry:\n analytics: false\n")
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_not_called()
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
self.raise_notice.assert_not_called()
|
||||
|
||||
def test_reversing_a_file_only_opt_out_resets_the_baseline(self):
|
||||
self.seed_state()
|
||||
self.save_config("telemetry:\n analytics: false\n")
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.save_config("telemetry:\n analytics: true\n")
|
||||
self.assert_reset_before_the_state_file()
|
||||
self.advance(WAKE_S)
|
||||
reporter.tick()
|
||||
|
||||
self.assertNotEqual(load_state(self.path).install_id, "a" * 32)
|
||||
self.registry.reset_analytics_baseline.assert_called_once()
|
||||
self.send.assert_called_once()
|
||||
|
||||
def test_yaml_truthy_spellings_in_the_file_count_as_consent(self):
|
||||
for value in ("yes", "on", "1", "'true'"):
|
||||
with self.subTest(value=value):
|
||||
self.seed_state()
|
||||
self.send.reset_mock()
|
||||
self.save_config(f"telemetry:\n analytics: {value}\n")
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once()
|
||||
self.assertTrue(os.path.exists(self.path))
|
||||
|
||||
def test_an_unreadable_config_file_falls_back_to_the_live_config(self):
|
||||
self.seed_state()
|
||||
self.save_config("telemetry: [unclosed\n")
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once()
|
||||
|
||||
def test_consent_withdrawn_while_the_report_builds_stops_the_send(self):
|
||||
reporter = self.reporter()
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
def withdraw(*args, **kwargs):
|
||||
self.holder.set(make_config())
|
||||
return self.built
|
||||
|
||||
self.build_report.side_effect = withdraw
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_not_called()
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
|
||||
def test_turning_sharing_off_and_on_between_wakes_starts_a_fresh_identity(self):
|
||||
self.seed_state()
|
||||
reporter = self.reporter()
|
||||
reporter.tick()
|
||||
|
||||
self.holder.set(make_config())
|
||||
self.holder.set(make_config({"telemetry": {"analytics": True}}))
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.raise_notice.assert_called_once_with(PROMPT_KIND)
|
||||
self.assertNotEqual(load_state(self.path).install_id, "a" * 32)
|
||||
|
||||
def test_opting_out_raises_the_prompt_once_and_deletes_the_state(self):
|
||||
self.holder.set(make_config())
|
||||
self.seed_state()
|
||||
reporter = self.reporter()
|
||||
|
||||
reporter.tick()
|
||||
reporter.tick()
|
||||
|
||||
self.raise_notice.assert_called_once_with(PROMPT_KIND)
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
self.send.assert_not_called()
|
||||
|
||||
def test_opting_in_resolves_the_prompt(self):
|
||||
self.holder.set(make_config())
|
||||
reporter = self.reporter()
|
||||
reporter.tick()
|
||||
|
||||
self.holder.set(make_config({"telemetry": {"analytics": True}}))
|
||||
reporter.tick()
|
||||
|
||||
self.resolve_notice.assert_called_once_with(PROMPT_KIND)
|
||||
|
||||
def test_safe_mode_touches_nothing(self):
|
||||
self.holder.set(make_config({"safe_mode": True}))
|
||||
self.seed_state()
|
||||
reporter = self.reporter()
|
||||
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.raise_notice.assert_not_called()
|
||||
self.assertTrue(os.path.exists(self.path))
|
||||
self.send.assert_not_called()
|
||||
|
||||
def test_an_unwritable_config_dir_skips_sending_and_warns_once(self):
|
||||
reporter = self.reporter(os.path.join(self.dir.name, "missing", "x.json"))
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
|
||||
with self.assertLogs("frigate.analytics.reporter", "WARNING") as logs:
|
||||
reporter.tick()
|
||||
self.advance(WAKE_S)
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_not_called()
|
||||
self.assertEqual(len(logs.output), 1)
|
||||
|
||||
def test_a_last_attempt_in_the_future_is_ignored(self):
|
||||
self.seed_state(self.now + 10 * INTERVAL_S)
|
||||
reporter = self.reporter()
|
||||
|
||||
self.advance(FIRST_DELAY_S[0])
|
||||
reporter.tick()
|
||||
|
||||
self.send.assert_called_once()
|
||||
|
||||
def test_the_loop_survives_a_failing_tick(self):
|
||||
reporter = self.reporter()
|
||||
reporter.stop_event.set()
|
||||
|
||||
with (
|
||||
patch.object(reporter, "tick", side_effect=RuntimeError("boom")),
|
||||
self.assertLogs("frigate.analytics.reporter", "ERROR"),
|
||||
):
|
||||
reporter.run()
|
||||
@@ -0,0 +1,137 @@
|
||||
"""Tests for the analytics report schema."""
|
||||
|
||||
import inspect
|
||||
import re
|
||||
import unittest
|
||||
from enum import Enum
|
||||
from typing import Any, get_args, get_origin
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
import frigate.detectors.hardware as detection_hardware
|
||||
import frigate.util.hwaccel as hwaccel
|
||||
from frigate.analytics.schema import (
|
||||
AnalyticsReport,
|
||||
DecodeFamily,
|
||||
HardwareKey,
|
||||
HwaccelKey,
|
||||
InputPresetKey,
|
||||
NoticeKindKey,
|
||||
)
|
||||
from frigate.ffmpeg_presets import PRESETS_HW_ACCEL_DECODE, PRESETS_INPUT
|
||||
from frigate.notices.types import NOTICE_KINDS
|
||||
from frigate.util.hwaccel import HwaccelFamily
|
||||
|
||||
# the only str fields: generated ids and strings the hardware reports
|
||||
ALLOWED_STRINGS = {
|
||||
("AnalyticsReport", "install_id"),
|
||||
("AnalyticsReport", "report_id"),
|
||||
("InstallSection", "version"),
|
||||
("InstallSection", "kernel"),
|
||||
("HardwareSection", "cpu_model"),
|
||||
("GpuInfo", "name"),
|
||||
("StorageInfo", "record_fs"),
|
||||
("DetectionModel", "input"),
|
||||
}
|
||||
|
||||
|
||||
def nested_models(annotation: Any) -> list[type[BaseModel]]:
|
||||
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
|
||||
return [annotation]
|
||||
|
||||
return [model for arg in get_args(annotation) for model in nested_models(arg)]
|
||||
|
||||
|
||||
def all_models(
|
||||
model: type[BaseModel], seen: set[type[BaseModel]] | None = None
|
||||
) -> list[type[BaseModel]]:
|
||||
seen = set() if seen is None else seen
|
||||
|
||||
if model in seen:
|
||||
return []
|
||||
|
||||
seen.add(model)
|
||||
found = [model]
|
||||
|
||||
for field in model.model_fields.values():
|
||||
for nested in nested_models(field.annotation):
|
||||
found += all_models(nested, seen)
|
||||
|
||||
return found
|
||||
|
||||
|
||||
def dict_key_types(annotation: Any) -> list[Any]:
|
||||
keys = [get_args(annotation)[0]] if get_origin(annotation) is dict else []
|
||||
return keys + [key for arg in get_args(annotation) for key in dict_key_types(arg)]
|
||||
|
||||
|
||||
class TestSchemaDiscipline(unittest.TestCase):
|
||||
def test_every_field_has_a_description_and_public_flag(self):
|
||||
for model in all_models(AnalyticsReport):
|
||||
for name, field in model.model_fields.items():
|
||||
with self.subTest(field=f"{model.__name__}.{name}"):
|
||||
self.assertTrue(field.description)
|
||||
self.assertIsInstance(field.json_schema_extra, dict)
|
||||
self.assertIsInstance(field.json_schema_extra.get("x-public"), bool)
|
||||
|
||||
def test_strings_are_limited_to_ids_and_hardware_names(self):
|
||||
for model in all_models(AnalyticsReport):
|
||||
properties = model.model_json_schema()["properties"]
|
||||
|
||||
for name, field in model.model_fields.items():
|
||||
if field.annotation is not str:
|
||||
continue
|
||||
|
||||
with self.subTest(field=f"{model.__name__}.{name}"):
|
||||
self.assertIn((model.__name__, name), ALLOWED_STRINGS)
|
||||
self.assertTrue(
|
||||
"maxLength" in properties[name] or "pattern" in properties[name]
|
||||
)
|
||||
|
||||
def test_map_keys_are_enums(self):
|
||||
for model in all_models(AnalyticsReport):
|
||||
for name, field in model.model_fields.items():
|
||||
for key_type in dict_key_types(field.annotation):
|
||||
with self.subTest(field=f"{model.__name__}.{name}"):
|
||||
self.assertTrue(issubclass(key_type, Enum))
|
||||
|
||||
|
||||
class TestEnumsTrackTheirSources(unittest.TestCase):
|
||||
def test_hardware_keys_cover_every_probe(self):
|
||||
probed = set(
|
||||
re.findall(
|
||||
r'_hardware\(\s*"([^"]+)"', inspect.getsource(detection_hardware)
|
||||
)
|
||||
)
|
||||
|
||||
self.assertTrue(probed)
|
||||
self.assertLessEqual(probed, {key.value for key in HardwareKey})
|
||||
|
||||
def test_decode_families_cover_every_hwaccel_family(self):
|
||||
families = {
|
||||
value.key
|
||||
for value in vars(hwaccel).values()
|
||||
if isinstance(value, HwaccelFamily)
|
||||
}
|
||||
|
||||
self.assertTrue(families)
|
||||
self.assertLessEqual(families, {family.value for family in DecodeFamily})
|
||||
|
||||
def test_preset_keys_mirror_the_presets(self):
|
||||
for enum, presets in (
|
||||
(HwaccelKey, PRESETS_HW_ACCEL_DECODE),
|
||||
(InputPresetKey, PRESETS_INPUT),
|
||||
):
|
||||
with self.subTest(enum=enum.__name__):
|
||||
self.assertEqual(
|
||||
{key.value for key in enum},
|
||||
{name.removeprefix("preset-") for name in presets}
|
||||
| {"custom", "none"},
|
||||
)
|
||||
|
||||
def test_notice_keys_are_the_reportable_kinds(self):
|
||||
self.assertEqual(
|
||||
{key.value for key in NoticeKindKey},
|
||||
{key for key, kind in NOTICE_KINDS.items() if kind.reportable},
|
||||
)
|
||||
self.assertNotIn("analytics_prompt", {key.value for key in NoticeKindKey})
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Tests for the analytics state file."""
|
||||
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from frigate.analytics.state import (
|
||||
AnalyticsState,
|
||||
delete_state,
|
||||
load_state,
|
||||
new_state,
|
||||
save_state,
|
||||
)
|
||||
|
||||
|
||||
class TestAnalyticsState(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.dir = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(self.dir.cleanup)
|
||||
self.path = os.path.join(self.dir.name, ".analytics.json")
|
||||
|
||||
def write(self, content: str) -> None:
|
||||
with open(self.path, "w") as f:
|
||||
f.write(content)
|
||||
|
||||
def test_missing_file_loads_as_none(self):
|
||||
self.assertIsNone(load_state(self.path))
|
||||
|
||||
def test_save_then_load_round_trips(self):
|
||||
state = AnalyticsState(install_id="a" * 32, last_attempt_at=1790000000.5)
|
||||
|
||||
self.assertTrue(save_state(state, self.path))
|
||||
self.assertEqual(load_state(self.path), state)
|
||||
|
||||
def test_unusable_files_load_as_none(self):
|
||||
for content in (
|
||||
"{not json",
|
||||
'["a"]',
|
||||
'{"install_id": "short", "last_attempt_at": 1}',
|
||||
'{"install_id": "' + "a" * 32 + '"}',
|
||||
'{"install_id": "' + "a" * 32 + '", "last_attempt_at": true}',
|
||||
):
|
||||
with self.subTest(content=content):
|
||||
self.write(content)
|
||||
self.assertIsNone(load_state(self.path))
|
||||
|
||||
def test_save_reports_failure_instead_of_raising(self):
|
||||
missing = os.path.join(self.dir.name, "missing", ".analytics.json")
|
||||
|
||||
self.assertFalse(save_state(new_state(), missing))
|
||||
|
||||
def test_new_state_has_a_hex_id_and_no_attempt(self):
|
||||
state = new_state()
|
||||
|
||||
self.assertRegex(state.install_id, r"^[0-9a-f]{32}$")
|
||||
self.assertEqual(state.last_attempt_at, 0.0)
|
||||
|
||||
def test_delete_removes_the_file_and_tolerates_a_missing_one(self):
|
||||
save_state(new_state(), self.path)
|
||||
|
||||
delete_state(self.path)
|
||||
delete_state(self.path)
|
||||
|
||||
self.assertFalse(os.path.exists(self.path))
|
||||
@@ -0,0 +1,64 @@
|
||||
"""Tests for sending a report."""
|
||||
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
import requests
|
||||
|
||||
from frigate.analytics.transport import TIMEOUT_S, SendOutcome, send_report
|
||||
from frigate.version import VERSION
|
||||
|
||||
URL = "https://example.test/report"
|
||||
|
||||
|
||||
def post(status: int = 204, text: str = "", side_effect: Exception | None = None):
|
||||
return patch(
|
||||
"frigate.analytics.transport.requests.post",
|
||||
return_value=Mock(status_code=status, text=text),
|
||||
side_effect=side_effect,
|
||||
)
|
||||
|
||||
|
||||
class TestSendReport(unittest.TestCase):
|
||||
def test_posts_the_body_as_json_with_a_user_agent(self):
|
||||
with post() as mock_post:
|
||||
outcome = send_report(URL, '{"a":1}')
|
||||
|
||||
self.assertIs(outcome, SendOutcome.accepted)
|
||||
args, kwargs = mock_post.call_args
|
||||
self.assertEqual(args[0], URL)
|
||||
self.assertEqual(kwargs["data"], b'{"a":1}')
|
||||
self.assertEqual(kwargs["headers"]["Content-Type"], "application/json")
|
||||
self.assertEqual(kwargs["headers"]["User-Agent"], f"Frigate/{VERSION}")
|
||||
self.assertFalse(kwargs["allow_redirects"])
|
||||
self.assertEqual(kwargs["timeout"], TIMEOUT_S)
|
||||
|
||||
def test_maps_statuses_to_outcomes(self):
|
||||
cases = (
|
||||
(200, SendOutcome.accepted),
|
||||
(400, SendOutcome.rejected),
|
||||
(429, SendOutcome.rate_limited),
|
||||
(302, SendOutcome.failed),
|
||||
(500, SendOutcome.failed),
|
||||
)
|
||||
|
||||
for status, outcome in cases:
|
||||
with self.subTest(status=status), post(status):
|
||||
self.assertIs(send_report(URL, "{}"), outcome)
|
||||
|
||||
def test_logs_the_rejection_reason(self):
|
||||
with (
|
||||
post(400, "install_id: bad"),
|
||||
self.assertLogs("frigate.analytics.transport", "DEBUG") as logs,
|
||||
):
|
||||
send_report(URL, "{}")
|
||||
|
||||
self.assertIn("install_id: bad", logs.output[0])
|
||||
|
||||
def test_a_network_error_fails_without_raising(self):
|
||||
# the user can't act on a failed send, so it stays out of warnings
|
||||
with (
|
||||
post(side_effect=requests.ConnectionError("down")),
|
||||
self.assertNoLogs("frigate.analytics.transport", "WARNING"),
|
||||
):
|
||||
self.assertIs(send_report(URL, "{}"), SendOutcome.failed)
|
||||
@@ -1,9 +1,10 @@
|
||||
"""Tests for frigate.util.builtin helpers."""
|
||||
|
||||
import unittest
|
||||
from multiprocessing import Value
|
||||
from unittest.mock import patch
|
||||
|
||||
from frigate.util.builtin import EventsPerSecond
|
||||
from frigate.util.builtin import EventsPerSecond, InferenceSpeed
|
||||
|
||||
|
||||
class TestEventsPerSecond(unittest.TestCase):
|
||||
@@ -62,5 +63,17 @@ class TestEventsPerSecond(unittest.TestCase):
|
||||
self.assertAlmostEqual(eps.eps(), 20.0)
|
||||
|
||||
|
||||
class TestInferenceSpeed(unittest.TestCase):
|
||||
def test_first_inference_seeds_the_average(self) -> None:
|
||||
metric = Value("d", 0.0)
|
||||
speed = InferenceSpeed(metric)
|
||||
|
||||
speed.update(0.02)
|
||||
self.assertEqual(metric.value, 0.02)
|
||||
|
||||
speed.update(0.12)
|
||||
self.assertAlmostEqual(metric.value, 0.03)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -18,6 +18,11 @@ from frigate.util.builtin import deep_merge
|
||||
|
||||
|
||||
class TestConfig(unittest.TestCase):
|
||||
def test_analytics_is_off_by_default(self):
|
||||
frigate_config = FrigateConfig(**self.minimal)
|
||||
|
||||
self.assertFalse(frigate_config.telemetry.analytics)
|
||||
|
||||
def setUp(self):
|
||||
self.minimal = {
|
||||
"mqtt": {"host": "mqtt"},
|
||||
|
||||
@@ -83,5 +83,30 @@ class TestSwapRuntimeConfig(unittest.TestCase):
|
||||
app.genai_manager.update_config.assert_called_once_with(config)
|
||||
|
||||
|
||||
class TestConfigHolder(unittest.TestCase):
|
||||
def test_set_passes_the_new_config_to_subscribers(self) -> None:
|
||||
holder = ConfigHolder(MagicMock(name="boot_config"))
|
||||
listener = MagicMock()
|
||||
holder.subscribe(listener)
|
||||
config = MagicMock(name="new_config")
|
||||
|
||||
holder.set(config)
|
||||
|
||||
listener.assert_called_once_with(config)
|
||||
|
||||
def test_a_failing_subscriber_does_not_block_the_swap(self) -> None:
|
||||
holder = ConfigHolder(MagicMock(name="boot_config"))
|
||||
after = MagicMock()
|
||||
holder.subscribe(MagicMock(side_effect=RuntimeError("boom")))
|
||||
holder.subscribe(after)
|
||||
config = MagicMock(name="new_config")
|
||||
|
||||
with self.assertLogs("frigate.config.holder", "ERROR"):
|
||||
holder.set(config)
|
||||
|
||||
self.assertIs(holder.config, config)
|
||||
after.assert_called_once_with(config)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -46,6 +46,16 @@ class TestNoticeKinds(unittest.TestCase):
|
||||
|
||||
self.assertIsNone(kind.link_for({"version": "0.19.1"}))
|
||||
|
||||
def test_analytics_prompt_links_to_telemetry_settings(self):
|
||||
kind = NOTICE_KINDS["analytics_prompt"]
|
||||
|
||||
self.assertEqual(kind.link_for({}), "/settings?page=systemTelemetry")
|
||||
self.assertFalse(kind.counts_repeats)
|
||||
self.assertFalse(kind.reportable)
|
||||
|
||||
def test_update_available_is_not_reported(self):
|
||||
self.assertFalse(NOTICE_KINDS["update_available"].reportable)
|
||||
|
||||
|
||||
class RegistryTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
@@ -347,6 +357,41 @@ class TestNoticeRegistry(RegistryTestCase):
|
||||
self.assertEqual(stats["skipped_detections"]["mutes"], 1)
|
||||
|
||||
|
||||
class TestAnalyticsBaseline(RegistryTestCase):
|
||||
def test_reset_counts_from_now_and_new_kinds_start_at_zero(self):
|
||||
stuck = {"scope": "ov", "params": {"detector": "ov"}}
|
||||
self.registry.raise_notice("detector_stuck", **stuck)
|
||||
self.registry.raise_notice("detector_stuck", **stuck)
|
||||
|
||||
self.registry.reset_analytics_baseline()
|
||||
self.registry.raise_notice("detector_stuck", **stuck)
|
||||
self.registry.raise_notice(
|
||||
"skipped_detections", scope="front_door", params={"pct": 12.5}
|
||||
)
|
||||
|
||||
rows = {row["kind"]: row for row in self.registry.stats()}
|
||||
self.assertEqual(
|
||||
(
|
||||
rows["detector_stuck"]["occurrences"],
|
||||
rows["detector_stuck"]["reported_occurrences"],
|
||||
),
|
||||
(3, 2),
|
||||
)
|
||||
self.assertEqual(rows["skipped_detections"]["reported_occurrences"], 0)
|
||||
|
||||
@patch.dict(NOTICE_KINDS, TEST_KINDS)
|
||||
def test_reset_puts_held_repeats_in_the_baseline(self):
|
||||
for _ in range(3):
|
||||
self.registry.raise_notice("batched")
|
||||
|
||||
self.registry.reset_analytics_baseline()
|
||||
self.registry.flush()
|
||||
|
||||
[row] = self.registry.stats()
|
||||
self.assertEqual((row["occurrences"], row["reported_occurrences"]), (3, 3))
|
||||
self.assertEqual(self.registry.active()[0]["count"], 3)
|
||||
|
||||
|
||||
class TestApply(RegistryTestCase):
|
||||
def test_each_action_reaches_its_method(self):
|
||||
self.registry.apply(
|
||||
|
||||
Reference in New Issue
Block a user