diff --git a/docs/static/frigate-api.yaml b/docs/static/frigate-api.yaml index 18a75df5da..7753ed7af3 100644 --- a/docs/static/frigate-api.yaml +++ b/docs/static/frigate-api.yaml @@ -4073,6 +4073,16 @@ paths: - type: 'null' default: 100 title: Limit + - name: offset + in: query + required: false + schema: + anyOf: + - type: integer + minimum: 0 + - type: 'null' + default: 0 + title: Offset - name: after in: query required: false @@ -4378,6 +4388,16 @@ paths: - type: 'null' default: 50 title: Limit + - name: offset + in: query + required: false + schema: + anyOf: + - type: integer + minimum: 0 + - type: 'null' + default: 0 + title: Offset - name: cameras in: query required: false diff --git a/frigate/api/defs/query/events_query_parameters.py b/frigate/api/defs/query/events_query_parameters.py index 06d0dfc3af..80cc77c721 100644 --- a/frigate/api/defs/query/events_query_parameters.py +++ b/frigate/api/defs/query/events_query_parameters.py @@ -14,6 +14,7 @@ class EventsQueryParams(BaseModel): zone: str | None = "all" zones: str | None = "all" limit: int | None = 100 + offset: int | None = Field(0, ge=0) after: float | None = None before: float | None = None time_range: str | None = DEFAULT_TIME_RANGE @@ -55,6 +56,7 @@ class EventsSearchQueryParams(BaseModel): deprecated=True, ) limit: int | None = 50 + offset: int | None = Field(0, ge=0) cameras: str | None = "all" labels: str | None = "all" sub_labels: str | None = "all" diff --git a/frigate/api/event.py b/frigate/api/event.py index 1597c18856..a0313902b1 100644 --- a/frigate/api/event.py +++ b/frigate/api/event.py @@ -129,6 +129,7 @@ def events( zones = zone limit = params.limit + offset = params.offset after = params.after before = params.before time_range = params.time_range @@ -361,11 +362,15 @@ def events( else: order_by = Event.start_time.desc() + # offset paging needs a stable order when scores or speeds tie + tiebreaker = [Event.id] if sort and sort.startswith(("score", "speed")) else [] + events = ( Event.select(*selected_columns) .where(reduce(operator.and_, clauses)) - .order_by(order_by) + .order_by(order_by, *tiebreaker) .limit(limit) + .offset(offset) .dicts() .iterator() ) @@ -518,6 +523,7 @@ def events_search( search_type = params.search_type include_thumbnails = params.include_thumbnails limit = params.limit + offset = params.offset sort = params.sort # Filters @@ -824,6 +830,9 @@ def events_search( if search_results: events_query = events_query.where(Event.id << list(search_results.keys())) + # sorts below are stable, so this orders ties for offset paging + events_query = events_query.order_by(Event.id) + # Fetch events and process them in a single pass processed_events = [] for event in events_query.dicts(): @@ -881,7 +890,7 @@ def events_search( processed_events.sort(key=lambda x: x["start_time"], reverse=True) # Limit the number of events returned - processed_events = processed_events[:limit] + processed_events = processed_events[offset:][:limit] return JSONResponse(content=processed_events) diff --git a/frigate/test/http_api/test_http_event.py b/frigate/test/http_api/test_http_event.py index 8aca6577d9..959fe6dbc2 100644 --- a/frigate/test/http_api/test_http_event.py +++ b/frigate/test/http_api/test_http_event.py @@ -168,6 +168,29 @@ class TestHttpApp(BaseTestHttp): assert events[0]["id"] == id assert events[1]["id"] == id2 + def test_get_event_list_offset_pages_score_sort(self): + now = datetime.now().timestamp() + scores = [0.6, 0.9, 0.7, 0.95, 0.8] + + with AuthTestClient(self.app) as client: + for i, score in enumerate(scores): + super().insert_mock_event( + f"event-{i}", start_time=now + i, data={"score": score} + ) + + params = {"sort": "score_desc"} + full = [e["id"] for e in client.get("/events", params=params).json()] + paged = [ + e["id"] + for offset in (0, 2, 4) + for e in client.get( + "/events", params={**params, "limit": 2, "offset": offset} + ).json() + ] + + assert full == ["event-3", "event-1", "event-4", "event-2", "event-0"] + assert paged == full + def test_get_event_list_match_multilingual_attribute(self): event_id = "123456.zh" attribute = "中文标签" @@ -219,6 +242,85 @@ class TestHttpApp(BaseTestHttp): assert len(events) == 1 assert events[0]["id"] == event_id + def test_events_search_offset_pages_score_sort(self): + now = datetime.now().timestamp() + scores = [0.6, 0.9, 0.7, 0.95, 0.8] + ids = [f"event-{i}" for i in range(len(scores))] + mock_embeddings = Mock() + mock_embeddings.search_thumbnail.return_value = [ + (event_id, 0.1 * i) for i, event_id in enumerate(ids) + ] + + self.app.frigate_config.semantic_search.enabled = True + self.app.embeddings = mock_embeddings + + with AuthTestClient(self.app) as client: + for i, score in enumerate(scores): + super().insert_mock_event( + ids[i], start_time=now + i, data={"score": score} + ) + + params = { + "search_type": "similarity", + "event_id": ids[0], + "sort": "score_desc", + } + paged = [ + e["id"] + for offset in (0, 2, 4) + for e in client.get( + "/events/search", + params={**params, "limit": 2, "offset": offset}, + ).json() + ] + + assert paged == ["event-3", "event-1", "event-4", "event-2", "event-0"] + + def test_events_search_offset_pages_orders_ties_by_id(self): + now = datetime.now().timestamp() + ids = ["event-c", "event-a", "event-b"] + mock_embeddings = Mock() + mock_embeddings.search_thumbnail.return_value = [ + (event_id, 0.1) for event_id in ids + ] + + self.app.frigate_config.semantic_search.enabled = True + self.app.embeddings = mock_embeddings + + with AuthTestClient(self.app) as client: + for i, event_id in enumerate(ids): + super().insert_mock_event( + event_id, start_time=now + i, data={"score": 0.8} + ) + + for sort in ("score_desc", "relevance"): + params = { + "search_type": "similarity", + "event_id": ids[0], + "sort": sort, + } + paged = [ + e["id"] + for offset in (0, 1, 2) + for e in client.get( + "/events/search", + params={**params, "limit": 1, "offset": offset}, + ).json() + ] + + assert paged == ["event-a", "event-b", "event-c"] + + def test_event_list_rejects_negative_offset(self): + with AuthTestClient(self.app) as client: + response = client.get("/events", params={"offset": -5}) + assert response.status_code == 422 + + response = client.get( + "/events/search", + params={"query": "car", "offset": -5}, + ) + assert response.status_code == 422 + def test_similarity_search_hides_unauthorized_anchor_event(self): mock_embeddings = Mock() self.app.frigate_config.semantic_search.enabled = True diff --git a/web/src/pages/Explore.tsx b/web/src/pages/Explore.tsx index b5b19c8afb..4b2ca38e5d 100644 --- a/web/src/pages/Explore.tsx +++ b/web/src/pages/Explore.tsx @@ -198,7 +198,19 @@ export default function Explore() { const [url, params] = searchQuery; - const isAscending = params.sort?.includes("date_asc"); + // a start_time cursor only works when rows are ordered by start_time, + // so every other sort pages by offset + const isDateSort = + params.sort === "date_asc" || + params.sort === "date_desc" || + (!params.sort && url === "events"); + + if (pageIndex > 0 && !isDateSort) { + return [ + url, + { ...params, offset: pageIndex * API_LIMIT, limit: API_LIMIT }, + ]; + } if (pageIndex > 0 && previousPageData) { const lastDate = previousPageData[previousPageData.length - 1].start_time; @@ -206,7 +218,8 @@ export default function Explore() { url, { ...params, - [isAscending ? "after" : "before"]: lastDate.toString(), + [params.sort === "date_asc" ? "after" : "before"]: + lastDate.toString(), limit: API_LIMIT, }, ]; @@ -238,10 +251,17 @@ export default function Explore() { }, }); - const searchResults = useMemo( - () => (data ? ([] as SearchResult[]).concat(...data) : []), - [data], - ); + // offset pages can overlap when results shift between page fetches + const searchResults = useMemo(() => { + if (!data) return []; + + const seen = new Set(); + return data.flat().filter((result) => { + if (seen.has(result.id)) return false; + seen.add(result.id); + return true; + }); + }, [data]); const isLoadingInitialData = !data && !isValidating; const isLoadingMore = isLoadingInitialData || diff --git a/web/src/types/search.ts b/web/src/types/search.ts index d47e955847..8740314bfc 100644 --- a/web/src/types/search.ts +++ b/web/src/types/search.ts @@ -109,6 +109,7 @@ export type SearchQueryParams = { max_speed?: number; search_type?: string; limit?: number; + offset?: number; in_progress?: number; include_thumbnails?: number; query?: string;