From 76373cbbe6d7580f718d06b44453d8bdd893bf35 Mon Sep 17 00:00:00 2001 From: iesad Date: Wed, 17 Sep 2025 00:59:08 -0600 Subject: [PATCH] pull count of detection events by label into prometheus metrics --- frigate/api/app.py | 8 +++- frigate/stats/prometheus.py | 49 ++++-------------------- frigate/test/http_api/test_http_event.py | 37 ++++++++++++++++++ frigate/test/test_storage.py | 7 ++-- 4 files changed, 55 insertions(+), 46 deletions(-) diff --git a/frigate/api/app.py b/frigate/api/app.py index d9e573d29..721a2edc5 100644 --- a/frigate/api/app.py +++ b/frigate/api/app.py @@ -7,6 +7,7 @@ import logging import os import traceback import urllib +from typing import Dict, Any, List from datetime import datetime, timedelta from functools import reduce from io import StringIO @@ -21,7 +22,7 @@ from fastapi.encoders import jsonable_encoder from fastapi.params import Depends from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse from markupsafe import escape -from peewee import SQL, operator +from peewee import SQL, operator, fn from pydantic import ValidationError from frigate.api.auth import require_role @@ -130,7 +131,10 @@ def metrics(request: Request): """Expose Prometheus metrics endpoint and update metrics with latest stats""" # Retrieve the latest statistics and update the Prometheus metrics stats = request.app.stats_emitter.get_latest_stats() - update_metrics(stats) + # query DB for count of events by camera, label + event_counts: List[Dict[str, Any]] = Event.select(Event.camera, Event.label, fn.Count()).group_by(Event.camera, Event.label).dicts() + + update_metrics(stats=stats, event_counts=event_counts) content, content_type = get_metrics() return Response(content=content, media_type=content_type) diff --git a/frigate/stats/prometheus.py b/frigate/stats/prometheus.py index bc545f21d..51ef78483 100644 --- a/frigate/stats/prometheus.py +++ b/frigate/stats/prometheus.py @@ -1,3 +1,4 @@ +from typing import Dict, Any, List import logging import re @@ -450,52 +451,16 @@ class CustomCollector(object): yield storage_total yield storage_used - # count events - events = [] - - if len(events) > 0: - # events[0] is newest event, last element is oldest, don't need to sort - - if not self.previous_event_id: - # ignore all previous events on startup, prometheus might have already counted them - self.previous_event_id = events[0]["id"] - self.previous_event_start_time = int(events[0]["start_time"]) - - for event in events: - # break if event already counted - if event["id"] == self.previous_event_id: - break - - # break if event starts before previous event - if event["start_time"] < self.previous_event_start_time: - break - - # store counted events in a dict - try: - cam = self.all_events[event["camera"]] - try: - cam[event["label"]] += 1 - except KeyError: - # create label dict if not exists - cam.update({event["label"]: 1}) - except KeyError: - # create camera and label dict if not exists - self.all_events.update({event["camera"]: {event["label"]: 1}}) - - # don't recount events next time - self.previous_event_id = events[0]["id"] - self.previous_event_start_time = int(events[0]["start_time"]) - camera_events = CounterMetricFamily( "frigate_camera_events", "Count of camera events since exporter started", labels=["camera", "label"], ) - for camera, cam_dict in self.all_events.items(): - for label, label_value in cam_dict.items(): - camera_events.add_metric([camera, label], label_value) - + if len(self.all_events) > 0: + for event_count in self.all_events: + camera_events.add_metric([event_count['camera'], event_count['label']], event_count['Count']) + yield camera_events @@ -503,7 +468,7 @@ collector = CustomCollector(None) REGISTRY.register(collector) -def update_metrics(stats): +def update_metrics(stats: Dict[str, Any], event_counts: List[Dict[str, Any]]): """Updates the Prometheus metrics with the given stats data.""" try: # Store the complete stats for later use by collect() @@ -512,6 +477,8 @@ def update_metrics(stats): # For backwards compatibility collector.process_stats = stats.copy() + collector.all_events = event_counts + # No need to call collect() here - it will be called by get_metrics() except Exception as e: logging.error(f"Error updating metrics: {e}") diff --git a/frigate/test/http_api/test_http_event.py b/frigate/test/http_api/test_http_event.py index 4ac4f458d..1b75a3276 100644 --- a/frigate/test/http_api/test_http_event.py +++ b/frigate/test/http_api/test_http_event.py @@ -9,6 +9,8 @@ from frigate.api.auth import get_allowed_cameras_for_filter, get_current_user from frigate.comms.event_metadata_updater import EventMetadataPublisher from frigate.models import Event, Recordings, ReviewSegment, Timeline from frigate.test.http_api.base_http_test import BaseTestHttp +from frigate.stats.emitter import StatsEmitter +from frigate.test.test_storage import _insert_mock_event class TestHttpApp(BaseTestHttp): @@ -293,3 +295,38 @@ class TestHttpApp(BaseTestHttp): sub_labels = client.get("/sub_labels").json() assert sub_labels assert sub_labels == [sub_label] + + #################################################################################################################### + ################################### GET /metrics Endpoint ######################################################### + #################################################################################################################### + def test_get_metrics(self): + """ensure correct prometheus metrics api response""" + with TestClient(self.app) as client: + ts_start = datetime.now().timestamp() + ts_end = ts_start + 30 + _insert_mock_event(id="abcde.random", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="01234.random", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="56789.random", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="101112.random", label="outside", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="131415.random", label="outside", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="161718.random", camera="porch", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="192021.random", camera="porch", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="222324.random", camera="porch", label="inside", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="252627.random", camera="porch", label="inside", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="282930.random", label="inside", start=ts_start, end=ts_end, retain=True) + _insert_mock_event(id="313233.random", label="inside", start=ts_start, end=ts_end, retain=True) + + stats_emitter = Mock(spec=StatsEmitter) + stats_emitter.get_latest_stats.return_value = self.test_stats + self.app.stats_emitter = stats_emitter + event = client.get(f"/metrics") + + assert "# TYPE frigate_detection_total_fps gauge" in event.text + assert "frigate_detection_total_fps 13.7" in event.text + assert "# HELP frigate_camera_events_total Count of camera events since exporter started" in event.text + assert "# TYPE frigate_camera_events_total counter" in event.text + assert 'frigate_camera_events_total{camera="front_door",label="Mock"} 3.0' in event.text + assert 'frigate_camera_events_total{camera="front_door",label="inside"} 2.0' in event.text + assert 'frigate_camera_events_total{camera="front_door",label="outside"} 2.0' in event.text + assert 'frigate_camera_events_total{camera="porch",label="Mock"} 2.0' in event.text + assert 'frigate_camera_events_total{camera="porch",label="inside"} 2.0' \ No newline at end of file diff --git a/frigate/test/test_storage.py b/frigate/test/test_storage.py index d36960f47..e5c2eb9b2 100644 --- a/frigate/test/test_storage.py +++ b/frigate/test/test_storage.py @@ -261,12 +261,13 @@ class TestHttp(unittest.TestCase): assert Recordings.get(Recordings.id == rec_k3_id) -def _insert_mock_event(id: str, start: int, end: int, retain: bool) -> Event: +def _insert_mock_event(id: str, start: int, end: int, retain: bool, camera: str = "front_door", + label: str = "Mock") -> Event: """Inserts a basic event model with a given id.""" return Event.insert( id=id, - label="Mock", - camera="front_door", + label=label, + camera=camera, start_time=start, end_time=end, top_score=100,