From 06967fec9171c9d4d766dcdcdd60da065b992589 Mon Sep 17 00:00:00 2001 From: Josh Hawkins <32435876+hawkeye217@users.noreply.github.com> Date: Thu, 17 Sep 2026 12:14:20 -0500 Subject: [PATCH] Fix explore paging for non-date sorts (#24392) * fix explore paging for non-date sorts Explore paged every sort by passing the last row's `start_time` as a `before` or `after` cursor, which only works when rows are ordered by `start_time`. For score, speed, and relevance sorts, each page dropped every match newer than that row and repeated older rows from earlier pages, so infinite scroll stopped after a few pages. `/events` and `/events/search` now accept `offset`, and Explore pages non-date sorts by offset. Date sorts keep the cursor because `useSWRInfinite` only revalidates the first page, and cursor keys for later pages follow it while offset keys don't. Score and speed sorts on `/events` break ties on `id` so offset pages stay stable. * order search ties by id and reject negative offsets `/events/search` sorted in Python over a query with no `ORDER BY`, so tied scores, speeds, or distances kept whatever order SQLite returned, which isn't guaranteed to match across page requests. The query is now ordered by id and the stable sorts keep that order for ties. `offset` also accepted negative values, which sliced from the end of the search results. --- docs/static/frigate-api.yaml | 20 ++++ .../api/defs/query/events_query_parameters.py | 2 + frigate/api/event.py | 13 ++- frigate/test/http_api/test_http_event.py | 102 ++++++++++++++++++ web/src/pages/Explore.tsx | 32 ++++-- web/src/types/search.ts | 1 + 6 files changed, 162 insertions(+), 8 deletions(-) diff --git a/docs/static/frigate-api.yaml b/docs/static/frigate-api.yaml index 18a75df5da..7753ed7af3 100644 --- a/docs/static/frigate-api.yaml +++ b/docs/static/frigate-api.yaml @@ -4073,6 +4073,16 @@ paths: - type: 'null' default: 100 title: Limit + - name: offset + in: query + required: false + schema: + anyOf: + - type: integer + minimum: 0 + - type: 'null' + default: 0 + title: Offset - name: after in: query required: false @@ -4378,6 +4388,16 @@ paths: - type: 'null' default: 50 title: Limit + - name: offset + in: query + required: false + schema: + anyOf: + - type: integer + minimum: 0 + - type: 'null' + default: 0 + title: Offset - name: cameras in: query required: false diff --git a/frigate/api/defs/query/events_query_parameters.py b/frigate/api/defs/query/events_query_parameters.py index 06d0dfc3af..80cc77c721 100644 --- a/frigate/api/defs/query/events_query_parameters.py +++ b/frigate/api/defs/query/events_query_parameters.py @@ -14,6 +14,7 @@ class EventsQueryParams(BaseModel): zone: str | None = "all" zones: str | None = "all" limit: int | None = 100 + offset: int | None = Field(0, ge=0) after: float | None = None before: float | None = None time_range: str | None = DEFAULT_TIME_RANGE @@ -55,6 +56,7 @@ class EventsSearchQueryParams(BaseModel): deprecated=True, ) limit: int | None = 50 + offset: int | None = Field(0, ge=0) cameras: str | None = "all" labels: str | None = "all" sub_labels: str | None = "all" diff --git a/frigate/api/event.py b/frigate/api/event.py index 1597c18856..a0313902b1 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -129,6 +129,7 @@ def events( zones = zone limit = params.limit + offset = params.offset after = params.after before = params.before time_range = params.time_range @@ -361,11 +362,15 @@ def events( else: order_by = Event.start_time.desc() + # offset paging needs a stable order when scores or speeds tie + tiebreaker = [Event.id] if sort and sort.startswith(("score", "speed")) else [] + events = ( Event.select(*selected_columns) .where(reduce(operator.and_, clauses)) - .order_by(order_by) + .order_by(order_by, *tiebreaker) .limit(limit) + .offset(offset) .dicts() .iterator() ) @@ -518,6 +523,7 @@ def events_search( search_type = params.search_type include_thumbnails = params.include_thumbnails limit = params.limit + offset = params.offset sort = params.sort # Filters @@ -824,6 +830,9 @@ def events_search( if search_results: events_query = events_query.where(Event.id << list(search_results.keys())) + # sorts below are stable, so this orders ties for offset paging + events_query = events_query.order_by(Event.id) + # Fetch events and process them in a single pass processed_events = [] for event in events_query.dicts(): @@ -881,7 +890,7 @@ def events_search( processed_events.sort(key=lambda x: x["start_time"], reverse=True) # Limit the number of events returned - processed_events = processed_events[:limit] + processed_events = processed_events[offset:][:limit] return JSONResponse(content=processed_events) diff --git a/frigate/test/http_api/test_http_event.py b/frigate/test/http_api/test_http_event.py index 8aca6577d9..959fe6dbc2 100644 --- a/frigate/test/http_api/test_http_event.py +++ b/frigate/test/http_api/test_http_event.py @@ -168,6 +168,29 @@ class TestHttpApp(BaseTestHttp): assert events[0]["id"] == id assert events[1]["id"] == id2 + def test_get_event_list_offset_pages_score_sort(self): + now = datetime.now().timestamp() + scores = [0.6, 0.9, 0.7, 0.95, 0.8] + + with AuthTestClient(self.app) as client: + for i, score in enumerate(scores): + super().insert_mock_event( + f"event-{i}", start_time=now + i, data={"score": score} + ) + + params = {"sort": "score_desc"} + full = [e["id"] for e in client.get("/events", params=params).json()] + paged = [ + e["id"] + for offset in (0, 2, 4) + for e in client.get( + "/events", params={**params, "limit": 2, "offset": offset} + ).json() + ] + + assert full == ["event-3", "event-1", "event-4", "event-2", "event-0"] + assert paged == full + def test_get_event_list_match_multilingual_attribute(self): event_id = "123456.zh" attribute = "中文标签" @@ -219,6 +242,85 @@ class TestHttpApp(BaseTestHttp): assert len(events) == 1 assert events[0]["id"] == event_id + def test_events_search_offset_pages_score_sort(self): + now = datetime.now().timestamp() + scores = [0.6, 0.9, 0.7, 0.95, 0.8] + ids = [f"event-{i}" for i in range(len(scores))] + mock_embeddings = Mock() + mock_embeddings.search_thumbnail.return_value = [ + (event_id, 0.1 * i) for i, event_id in enumerate(ids) + ] + + self.app.frigate_config.semantic_search.enabled = True + self.app.embeddings = mock_embeddings + + with AuthTestClient(self.app) as client: + for i, score in enumerate(scores): + super().insert_mock_event( + ids[i], start_time=now + i, data={"score": score} + ) + + params = { + "search_type": "similarity", + "event_id": ids[0], + "sort": "score_desc", + } + paged = [ + e["id"] + for offset in (0, 2, 4) + for e in client.get( + "/events/search", + params={**params, "limit": 2, "offset": offset}, + ).json() + ] + + assert paged == ["event-3", "event-1", "event-4", "event-2", "event-0"] + + def test_events_search_offset_pages_orders_ties_by_id(self): + now = datetime.now().timestamp() + ids = ["event-c", "event-a", "event-b"] + mock_embeddings = Mock() + mock_embeddings.search_thumbnail.return_value = [ + (event_id, 0.1) for event_id in ids + ] + + self.app.frigate_config.semantic_search.enabled = True + self.app.embeddings = mock_embeddings + + with AuthTestClient(self.app) as client: + for i, event_id in enumerate(ids): + super().insert_mock_event( + event_id, start_time=now + i, data={"score": 0.8} + ) + + for sort in ("score_desc", "relevance"): + params = { + "search_type": "similarity", + "event_id": ids[0], + "sort": sort, + } + paged = [ + e["id"] + for offset in (0, 1, 2) + for e in client.get( + "/events/search", + params={**params, "limit": 1, "offset": offset}, + ).json() + ] + + assert paged == ["event-a", "event-b", "event-c"] + + def test_event_list_rejects_negative_offset(self): + with AuthTestClient(self.app) as client: + response = client.get("/events", params={"offset": -5}) + assert response.status_code == 422 + + response = client.get( + "/events/search", + params={"query": "car", "offset": -5}, + ) + assert response.status_code == 422 + def test_similarity_search_hides_unauthorized_anchor_event(self): mock_embeddings = Mock() self.app.frigate_config.semantic_search.enabled = True diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index b5b19c8afb..4b2ca38e5d 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -198,7 +198,19 @@ export default function Explore() { const [url, params] = searchQuery; - const isAscending = params.sort?.includes("date_asc"); + // a start_time cursor only works when rows are ordered by start_time, + // so every other sort pages by offset + const isDateSort = + params.sort === "date_asc" || + params.sort === "date_desc" || + (!params.sort && url === "events"); + + if (pageIndex > 0 && !isDateSort) { + return [ + url, + { ...params, offset: pageIndex * API_LIMIT, limit: API_LIMIT }, + ]; + } if (pageIndex > 0 && previousPageData) { const lastDate = previousPageData[previousPageData.length - 1].start_time; @@ -206,7 +218,8 @@ export default function Explore() { url, { ...params, - [isAscending ? "after" : "before"]: lastDate.toString(), + [params.sort === "date_asc" ? "after" : "before"]: + lastDate.toString(), limit: API_LIMIT, }, ]; @@ -238,10 +251,17 @@ export default function Explore() { }, }); - const searchResults = useMemo( - () => (data ? ([] as SearchResult[]).concat(...data) : []), - [data], - ); + // offset pages can overlap when results shift between page fetches + const searchResults = useMemo(() => { + if (!data) return []; + + const seen = new Set(); + return data.flat().filter((result) => { + if (seen.has(result.id)) return false; + seen.add(result.id); + return true; + }); + }, [data]); const isLoadingInitialData = !data && !isValidating; const isLoadingMore = isLoadingInitialData || diff --git a/web/src/types/search.ts b/web/src/types/search.ts index d47e955847..8740314bfc 100644 --- a/web/src/types/search.ts +++ b/web/src/types/search.ts @@ -109,6 +109,7 @@ export type SearchQueryParams = { max_speed?: number; search_type?: string; limit?: number; + offset?: number; in_progress?: number; include_thumbnails?: number; query?: string;