Files
frigate/frigate/test/test_analytics_report.py
T
Josh HawkinsandGitHub 4eb60af5a7 Implement opt-in analytics (#24440)
* Add analytics opt-in setting, prompt notice kind, and notice watermarks

* Add the analytics report schema

* Add the analytics state file

* Add the install and hardware analytics collectors

* Add the detection analytics collector

* Add the cameras analytics collector

* Add the features analytics collector

* Add the health analytics collector

* Build analytics reports from the section collectors

* Add the analytics report transport

* Send the daily analytics report from the main process

* Add the analytics preview API

* Generate the analytics schema and check it in CI

* Record the image variant in each Dockerfile

* Show the analytics preview in telemetry settings

* Document anonymous analytics

* change wording

* fixes

* update network requirements docs

* adapt analytics to free-form model scenes, split notice counts, and peewee 4

* fixes

* change log levels

* docstrings

* docstrings

* honor file-only analytics opt-outs and skip pre-consent notices

An opt-out saved through the YAML editor's "Save only" or a hand edit never reached the live config, so reports kept going out under the old install ID until a restart. The reporter now reads telemetry.analytics from the config file when a report is due, validated with TelemetryConfig so yes, on, and 1 still count as consent, and an opt-out there deletes the state file and skips the send. The first report under a new install ID also carried every notice since the counters started, including ones from before the opt-in, so a fresh state file now resets the notice watermarks before the report's snapshot.

* take the notice baseline at opt-in instead of the first send

The watermark reset ran at the first send attempt, so notices raised between the opt-in and that send, up to 45 minutes after a startup, were absorbed into the reset and never reported. The reporter now writes the new install ID and resets the watermarks as soon as it sees consent without a state file.

* report notice counts cumulatively from a database baseline

Moving watermarks after each accepted send kept the install ID and the notice baseline out of step: pre-consent notices went out, notices raised before the first send were dropped, and a send that finished after an off/on toggle overwrote the new identity's baseline. health.notices now carries cumulative counts since the install ID was created. Creating an ID copies the lifetime counts into the existing reported_* columns before the state file names it, and nothing is written after a send, so the backend diffs consecutive reports per install ID.

* write held notice repeats before the analytics baseline reset

Repeats of batched kinds like failed_login wait in memory until the stats emitter flushes them, so a baseline reset taken while some were held left them out, and the next flush counted those pre-opt-in repeats as post-consent. The reset now writes held repeats to the notice history under the same lock before copying the counts.

* seed detector inference speed from the first inference

The average started at 0 so idle detectors read 0, but that made every detector's speed climb from 0 for its first couple dozen inferences after a start. Detectors now use the same InferenceSpeed helper as enrichments, so the first inference seeds the average and idle detectors still read 0.
2026-11-06 07:43:38 -07:00

144 lines
4.7 KiB
Python

"""Tests for building a report from the collectors."""
import json
import logging
import os
import unittest
from unittest.mock import Mock, patch
from peewee import SqliteDatabase
from peewee_migrate import Router
from playhouse.sqliteq import SqliteQueueDatabase
from frigate.analytics import report
from frigate.analytics.collectors import hardware
from frigate.analytics.schema import AnalyticsReport, InstallSection
from frigate.analytics.state import AnalyticsState
from frigate.models import User
from frigate.test.analytics_helpers import make_context
from frigate.test.const import TEST_DB, TEST_DB_CLEANUPS
INSTALL = InstallSection(
version="0.19.0",
image_variant="dev",
install_type="docker",
arch="x86_64",
kernel="6.8",
run_as_root=False,
)
class TestBuildReport(unittest.TestCase):
def test_a_failing_collector_nulls_only_its_section(self):
collectors = {
"install": lambda ctx: INSTALL,
"hardware": Mock(side_effect=RuntimeError("boom")),
"detection": lambda ctx: None,
"cameras": lambda ctx: None,
"features": lambda ctx: None,
"health": lambda ctx: None,
}
with patch.dict(report.COLLECTORS, collectors):
built = report.build_report(make_context(), "a" * 32, sent_at=1790000000)
self.assertEqual(built.install, INSTALL)
self.assertIsNone(built.hardware)
self.assertEqual(built.schema_version, 1)
self.assertEqual(built.sent_at, 1790000000)
self.assertRegex(built.report_id, r"^[0-9a-f-]{36}$")
class TestRealCollectors(unittest.TestCase):
def setUp(self):
migrate_db = SqliteDatabase(TEST_DB)
del logging.getLogger("peewee_migrate").handlers[:]
Router(migrate_db).run()
migrate_db.close()
self.db = SqliteQueueDatabase(TEST_DB)
self.db.bind([User])
def tearDown(self):
# close() leaves the queue's writer thread running against the deleted file
self.db.stop()
if not self.db.is_closed():
self.db.close()
for file in TEST_DB_CLEANUPS:
try:
os.remove(file)
except OSError:
pass
def test_every_section_builds_and_round_trips_through_json(self):
with (
patch.object(hardware.hardware_prober, "probe", return_value=[]),
patch.object(hardware, "hwaccel_options", return_value=("", [])),
):
built = report.build_report(make_context(), "a" * 32)
body = built.model_dump_json()
parsed = json.loads(body)
for section in (
"install",
"hardware",
"detection",
"cameras",
"features",
"health",
):
with self.subTest(section=section):
self.assertIsNotNone(parsed[section])
self.assertEqual(AnalyticsReport.model_validate_json(body), built)
def test_preview_uses_a_placeholder_id_before_the_first_report(self):
stats_emitter = Mock()
stats_emitter.get_latest_stats.return_value = {}
with (
patch.object(report, "load_state", return_value=None),
patch.object(hardware.hardware_prober, "probe", return_value=[]),
patch.object(hardware, "hwaccel_options", return_value=("", [])),
):
built = report.preview_report(make_context().config, stats_emitter, None)
self.assertEqual(built.install_id, report.PREVIEW_INSTALL_ID)
def test_preview_counts_notices_only_once_opted_in(self):
stats_emitter = Mock()
stats_emitter.get_latest_stats.return_value = {}
registry = Mock()
registry.stats.return_value = [
{
"kind": "detector_stuck",
"occurrences": 3,
"acknowledgements": 0,
"mutes": 0,
"reported_occurrences": 1,
"reported_acknowledgements": 0,
"reported_mutes": 0,
}
]
for state, expected in (
(None, {}),
(AnalyticsState("a" * 32, 0.0), {"detector_stuck": 2}),
):
with (
self.subTest(opted_in=state is not None),
patch.object(report, "load_state", return_value=state),
patch.object(hardware.hardware_prober, "probe", return_value=[]),
patch.object(hardware, "hwaccel_options", return_value=("", [])),
):
built = report.preview_report(
make_context().config, stats_emitter, registry
)
self.assertEqual(
{kind: c.occurrences for kind, c in built.health.notices.items()},
expected,
)