pull count of detection events by label into prometheus metrics

frigate/api/app.py

@@ -7,6 +7,7 @@ import logging
 import os
 import traceback
 import urllib
+from typing import Dict, Any, List
 from datetime import datetime, timedelta
 from functools import reduce
 from io import StringIO
@@ -21,7 +22,7 @@ from fastapi.encoders import jsonable_encoder
 from fastapi.params import Depends
 from fastapi.responses import JSONResponse, PlainTextResponse, StreamingResponse
 from markupsafe import escape
-from peewee import SQL, operator
+from peewee import SQL, operator, fn
 from pydantic import ValidationError
 
 from frigate.api.auth import require_role
@@ -130,7 +131,10 @@ def metrics(request: Request):
     """Expose Prometheus metrics endpoint and update metrics with latest stats"""
     # Retrieve the latest statistics and update the Prometheus metrics
     stats = request.app.stats_emitter.get_latest_stats()
-    update_metrics(stats)
+    # query DB for count of events by camera, label
+    event_counts: List[Dict[str, Any]] = Event.select(Event.camera, Event.label, fn.Count()).group_by(Event.camera, Event.label).dicts()
+    
+    update_metrics(stats=stats, event_counts=event_counts)
     content, content_type = get_metrics()
     return Response(content=content, media_type=content_type)
 

frigate/stats/prometheus.py

@@ -1,3 +1,4 @@
+from typing import Dict, Any, List
 import logging
 import re
 
@@ -450,52 +451,16 @@ class CustomCollector(object):
         yield storage_total
         yield storage_used
 
-        # count events
-        events = []
-
-        if len(events) > 0:
-            # events[0] is newest event, last element is oldest, don't need to sort
-
-            if not self.previous_event_id:
-                # ignore all previous events on startup, prometheus might have already counted them
-                self.previous_event_id = events[0]["id"]
-                self.previous_event_start_time = int(events[0]["start_time"])
-
-            for event in events:
-                # break if event already counted
-                if event["id"] == self.previous_event_id:
-                    break
-
-                # break if event starts before previous event
-                if event["start_time"] < self.previous_event_start_time:
-                    break
-
-                # store counted events in a dict
-                try:
-                    cam = self.all_events[event["camera"]]
-                    try:
-                        cam[event["label"]] += 1
-                    except KeyError:
-                        # create label dict if not exists
-                        cam.update({event["label"]: 1})
-                except KeyError:
-                    # create camera and label dict if not exists
-                    self.all_events.update({event["camera"]: {event["label"]: 1}})
-
-            # don't recount events next time
-            self.previous_event_id = events[0]["id"]
-            self.previous_event_start_time = int(events[0]["start_time"])
-
         camera_events = CounterMetricFamily(
             "frigate_camera_events",
             "Count of camera events since exporter started",
             labels=["camera", "label"],
         )
 
-        for camera, cam_dict in self.all_events.items():
+        if len(self.all_events) > 0:
-            for label, label_value in cam_dict.items():
+            for event_count in self.all_events:
-                camera_events.add_metric([camera, label], label_value)
+                camera_events.add_metric([event_count['camera'], event_count['label']], event_count['Count'])
-
+        
         yield camera_events
 
 
@@ -503,7 +468,7 @@ collector = CustomCollector(None)
 REGISTRY.register(collector)
 
 
-def update_metrics(stats):
+def update_metrics(stats: Dict[str, Any], event_counts: List[Dict[str, Any]]):
     """Updates the Prometheus metrics with the given stats data."""
     try:
         # Store the complete stats for later use by collect()
@@ -512,6 +477,8 @@ def update_metrics(stats):
         # For backwards compatibility
         collector.process_stats = stats.copy()
 
+        collector.all_events = event_counts
+
         # No need to call collect() here - it will be called by get_metrics()
     except Exception as e:
         logging.error(f"Error updating metrics: {e}")

frigate/test/http_api/test_http_event.py

@@ -9,6 +9,8 @@ from frigate.api.auth import get_allowed_cameras_for_filter, get_current_user
 from frigate.comms.event_metadata_updater import EventMetadataPublisher
 from frigate.models import Event, Recordings, ReviewSegment, Timeline
 from frigate.test.http_api.base_http_test import BaseTestHttp
+from frigate.stats.emitter import StatsEmitter
+from frigate.test.test_storage import _insert_mock_event
 
 
 class TestHttpApp(BaseTestHttp):
@@ -293,3 +295,38 @@ class TestHttpApp(BaseTestHttp):
             sub_labels = client.get("/sub_labels").json()
             assert sub_labels
             assert sub_labels == [sub_label]
+
+    ####################################################################################################################
+    ###################################  GET /metrics Endpoint   #########################################################
+    ####################################################################################################################
+    def test_get_metrics(self):
+        """ensure correct prometheus metrics api response"""
+        with TestClient(self.app) as client:
+            ts_start = datetime.now().timestamp()
+            ts_end = ts_start + 30
+            _insert_mock_event(id="abcde.random", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="01234.random", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="56789.random", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="101112.random", label="outside", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="131415.random", label="outside", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="161718.random", camera="porch", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="192021.random", camera="porch", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="222324.random", camera="porch", label="inside", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="252627.random", camera="porch", label="inside", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="282930.random", label="inside", start=ts_start, end=ts_end, retain=True)
+            _insert_mock_event(id="313233.random", label="inside", start=ts_start, end=ts_end, retain=True)
+            
+            stats_emitter = Mock(spec=StatsEmitter)
+            stats_emitter.get_latest_stats.return_value = self.test_stats
+            self.app.stats_emitter = stats_emitter
+            event = client.get(f"/metrics")
+            
+        assert "# TYPE frigate_detection_total_fps gauge" in event.text
+        assert "frigate_detection_total_fps 13.7"         in event.text
+        assert "# HELP frigate_camera_events_total Count of camera events since exporter started"   in event.text
+        assert "# TYPE frigate_camera_events_total counter"                                         in event.text
+        assert 'frigate_camera_events_total{camera="front_door",label="Mock"} 3.0'                  in event.text
+        assert 'frigate_camera_events_total{camera="front_door",label="inside"} 2.0'                in event.text
+        assert 'frigate_camera_events_total{camera="front_door",label="outside"} 2.0'               in event.text
+        assert 'frigate_camera_events_total{camera="porch",label="Mock"} 2.0'                       in event.text
+        assert 'frigate_camera_events_total{camera="porch",label="inside"} 2.0'   

frigate/test/test_storage.py

@@ -261,12 +261,13 @@ class TestHttp(unittest.TestCase):
         assert Recordings.get(Recordings.id == rec_k3_id)
 
 
-def _insert_mock_event(id: str, start: int, end: int, retain: bool) -> Event:
+def _insert_mock_event(id: str, start: int, end: int, retain: bool, camera: str = "front_door",
+    label: str = "Mock") -> Event:
     """Inserts a basic event model with a given id."""
     return Event.insert(
         id=id,
-        label="Mock",
+        label=label,
-        camera="front_door",
+        camera=camera,
         start_time=start,
         end_time=end,
         top_score=100,