Misc backend performance improvements (#23244)

* Add additional indicies on event and review tables.  Every events or timeline endpoint filters on event start time and camera, this should speed things up by avoiding a range scan on the table.

* Rewrite to use a CTE to leverage speedups by using sqllite internal optimization to do a single query instead of a starter query to get distinct labels and a subsequent loop of querys per distinct event labels.

Frigate is currently shipping sqlite 3.46.1, which is above the minimum version 3.25 needed for CTEs.

* Collapse a few sequential queries into a single one.

* Use peewee instead of rw sql for the CTE query.

* Slightly simplify review logic and avoid duplicating the json response for empty review IDs.

* Rerun ruff formatting.

* Remove 2x unnecessary index on reviewsegment, remove reference to prior code implementation in comment in event.py

* Editor fail, re-ruff format.

* Remove the CTE and restore the generator with sub-queries, which is more performance (thanks Nick and Blake for testing against your larger DB!)

* Update peewee index migration description

* Add on_conflict_ignore, replacing the try/catch/pass on IntegrityError

* Add a testcase for validating that on_conflict_ignore bypasses what was formerly an IntegrityError

* Change testcase to clarify that it covers the peewee behavior of on_conflict_ignore

---------

Co-authored-by: Greg <{ID}+{username}@users.noreply.github.com>
This commit is contained in:
gwmullin
2026-09-03 15:53:59 -06:00
committed by GitHub
co-authored by Greg
parent 39f76e4a2c
commit 2961449d4d
4 changed files with 147 additions and 46 deletions
+28 -12
View File
@@ -386,7 +386,9 @@ def events_explore(
limit: int = 10,
allowed_cameras: list[str] = Depends(get_allowed_cameras_for_filter),
):
# get distinct labels for all events
if not allowed_cameras:
return JSONResponse(content=[])
distinct_labels = (
Event.select(Event.label)
.where(Event.camera << allowed_cameras)
@@ -396,13 +398,31 @@ def events_explore(
label_counts = {}
explore_columns = (
Event.id,
Event.camera,
Event.label,
Event.sub_label,
Event.zones,
Event.start_time,
Event.end_time,
Event.has_clip,
Event.has_snapshot,
Event.plus_id,
Event.retain_indefinitely,
Event.top_score,
Event.false_positive,
Event.box,
Event.data,
)
def event_generator():
for label_obj in distinct_labels.iterator():
label = label_obj.label
# get most recent events for this label
label_events = (
Event.select()
Event.select(*explore_columns)
.where((Event.label == label) & (Event.camera << allowed_cameras))
.order_by(Event.start_time.desc())
.limit(limit)
@@ -484,22 +504,18 @@ async def event_ids(ids: str, request: Request):
status_code=400,
)
for event_id in ids:
try:
event = Event.get(Event.id == event_id)
await require_camera_access(event.camera, request=request)
except DoesNotExist:
# we should not fail the entire request if an event is not found
continue
try:
events = Event.select().where(Event.id << ids).dicts().iterator()
return JSONResponse(list(events))
events = list(Event.select().where(Event.id << ids).dicts().iterator())
except Exception:
return JSONResponse(
content=({"success": False, "message": "Events not found"}), status_code=400
)
for event in events:
await require_camera_access(event["camera"], request=request)
return JSONResponse(events)
@router.get(
"/events/search",