Fix explore paging for non-date sorts (#24392)

* fix explore paging for non-date sorts

Explore paged every sort by passing the last row's `start_time` as a `before` or `after` cursor, which only works when rows are ordered by `start_time`. For score, speed, and relevance sorts, each page dropped every match newer than that row and repeated older rows from earlier pages, so infinite scroll stopped after a few pages. `/events` and `/events/search` now accept `offset`, and Explore pages non-date sorts by offset. Date sorts keep the cursor because `useSWRInfinite` only revalidates the first page, and cursor keys for later pages follow it while offset keys don't. Score and speed sorts on `/events` break ties on `id` so offset pages stay stable.

* order search ties by id and reject negative offsets

`/events/search` sorted in Python over a query with no `ORDER BY`, so tied scores, speeds, or distances kept whatever order SQLite returned, which isn't guaranteed to match across page requests. The query is now ordered by id and the stable sorts keep that order for ties. `offset` also accepted negative values, which sliced from the end of the search results.

docs/static/frigate-api.yaml

@@ -4073,6 +4073,16 @@ paths:
               - type: 'null'
             default: 100
             title: Limit
+        - name: offset
+          in: query
+          required: false
+          schema:
+            anyOf:
+              - type: integer
+                minimum: 0
+              - type: 'null'
+            default: 0
+            title: Offset
         - name: after
           in: query
           required: false
@@ -4378,6 +4388,16 @@ paths:
               - type: 'null'
             default: 50
             title: Limit
+        - name: offset
+          in: query
+          required: false
+          schema:
+            anyOf:
+              - type: integer
+                minimum: 0
+              - type: 'null'
+            default: 0
+            title: Offset
         - name: cameras
           in: query
           required: false

frigate/api/defs/query/events_query_parameters.py

@@ -14,6 +14,7 @@ class EventsQueryParams(BaseModel):
     zone: str | None = "all"
     zones: str | None = "all"
     limit: int | None = 100
+    offset: int | None = Field(0, ge=0)
     after: float | None = None
     before: float | None = None
     time_range: str | None = DEFAULT_TIME_RANGE
@@ -55,6 +56,7 @@ class EventsSearchQueryParams(BaseModel):
         deprecated=True,
     )
     limit: int | None = 50
+    offset: int | None = Field(0, ge=0)
     cameras: str | None = "all"
     labels: str | None = "all"
     sub_labels: str | None = "all"

frigate/api/event.py

@@ -129,6 +129,7 @@ def events(
         zones = zone
 
     limit = params.limit
+    offset = params.offset
     after = params.after
     before = params.before
     time_range = params.time_range
@@ -361,11 +362,15 @@ def events(
     else:
         order_by = Event.start_time.desc()
 
+    # offset paging needs a stable order when scores or speeds tie
+    tiebreaker = [Event.id] if sort and sort.startswith(("score", "speed")) else []
+
     events = (
         Event.select(*selected_columns)
         .where(reduce(operator.and_, clauses))
-        .order_by(order_by)
+        .order_by(order_by, *tiebreaker)
         .limit(limit)
+        .offset(offset)
         .dicts()
         .iterator()
     )
@@ -518,6 +523,7 @@ def events_search(
     search_type = params.search_type
     include_thumbnails = params.include_thumbnails
     limit = params.limit
+    offset = params.offset
     sort = params.sort
 
     # Filters
@@ -824,6 +830,9 @@ def events_search(
     if search_results:
         events_query = events_query.where(Event.id << list(search_results.keys()))
 
+    # sorts below are stable, so this orders ties for offset paging
+    events_query = events_query.order_by(Event.id)
+
     # Fetch events and process them in a single pass
     processed_events = []
     for event in events_query.dicts():
@@ -881,7 +890,7 @@ def events_search(
         processed_events.sort(key=lambda x: x["start_time"], reverse=True)
 
     # Limit the number of events returned
-    processed_events = processed_events[:limit]
+    processed_events = processed_events[offset:][:limit]
 
     return JSONResponse(content=processed_events)
 

frigate/test/http_api/test_http_event.py

@@ -168,6 +168,29 @@ class TestHttpApp(BaseTestHttp):
             assert events[0]["id"] == id
             assert events[1]["id"] == id2
 
+    def test_get_event_list_offset_pages_score_sort(self):
+        now = datetime.now().timestamp()
+        scores = [0.6, 0.9, 0.7, 0.95, 0.8]
+
+        with AuthTestClient(self.app) as client:
+            for i, score in enumerate(scores):
+                super().insert_mock_event(
+                    f"event-{i}", start_time=now + i, data={"score": score}
+                )
+
+            params = {"sort": "score_desc"}
+            full = [e["id"] for e in client.get("/events", params=params).json()]
+            paged = [
+                e["id"]
+                for offset in (0, 2, 4)
+                for e in client.get(
+                    "/events", params={**params, "limit": 2, "offset": offset}
+                ).json()
+            ]
+
+            assert full == ["event-3", "event-1", "event-4", "event-2", "event-0"]
+            assert paged == full
+
     def test_get_event_list_match_multilingual_attribute(self):
         event_id = "123456.zh"
         attribute = "中文标签"
@@ -219,6 +242,85 @@ class TestHttpApp(BaseTestHttp):
             assert len(events) == 1
             assert events[0]["id"] == event_id
 
+    def test_events_search_offset_pages_score_sort(self):
+        now = datetime.now().timestamp()
+        scores = [0.6, 0.9, 0.7, 0.95, 0.8]
+        ids = [f"event-{i}" for i in range(len(scores))]
+        mock_embeddings = Mock()
+        mock_embeddings.search_thumbnail.return_value = [
+            (event_id, 0.1 * i) for i, event_id in enumerate(ids)
+        ]
+
+        self.app.frigate_config.semantic_search.enabled = True
+        self.app.embeddings = mock_embeddings
+
+        with AuthTestClient(self.app) as client:
+            for i, score in enumerate(scores):
+                super().insert_mock_event(
+                    ids[i], start_time=now + i, data={"score": score}
+                )
+
+            params = {
+                "search_type": "similarity",
+                "event_id": ids[0],
+                "sort": "score_desc",
+            }
+            paged = [
+                e["id"]
+                for offset in (0, 2, 4)
+                for e in client.get(
+                    "/events/search",
+                    params={**params, "limit": 2, "offset": offset},
+                ).json()
+            ]
+
+            assert paged == ["event-3", "event-1", "event-4", "event-2", "event-0"]
+
+    def test_events_search_offset_pages_orders_ties_by_id(self):
+        now = datetime.now().timestamp()
+        ids = ["event-c", "event-a", "event-b"]
+        mock_embeddings = Mock()
+        mock_embeddings.search_thumbnail.return_value = [
+            (event_id, 0.1) for event_id in ids
+        ]
+
+        self.app.frigate_config.semantic_search.enabled = True
+        self.app.embeddings = mock_embeddings
+
+        with AuthTestClient(self.app) as client:
+            for i, event_id in enumerate(ids):
+                super().insert_mock_event(
+                    event_id, start_time=now + i, data={"score": 0.8}
+                )
+
+            for sort in ("score_desc", "relevance"):
+                params = {
+                    "search_type": "similarity",
+                    "event_id": ids[0],
+                    "sort": sort,
+                }
+                paged = [
+                    e["id"]
+                    for offset in (0, 1, 2)
+                    for e in client.get(
+                        "/events/search",
+                        params={**params, "limit": 1, "offset": offset},
+                    ).json()
+                ]
+
+                assert paged == ["event-a", "event-b", "event-c"]
+
+    def test_event_list_rejects_negative_offset(self):
+        with AuthTestClient(self.app) as client:
+            response = client.get("/events", params={"offset": -5})
+            assert response.status_code == 422
+
+            response = client.get(
+                "/events/search",
+                params={"query": "car", "offset": -5},
+            )
+            assert response.status_code == 422
+
     def test_similarity_search_hides_unauthorized_anchor_event(self):
         mock_embeddings = Mock()
         self.app.frigate_config.semantic_search.enabled = True

web/src/pages/Explore.tsx

@@ -198,7 +198,19 @@ export default function Explore() {
 
     const [url, params] = searchQuery;
 
-    const isAscending = params.sort?.includes("date_asc");
+    // a start_time cursor only works when rows are ordered by start_time,
+    // so every other sort pages by offset
+    const isDateSort =
+      params.sort === "date_asc" ||
+      params.sort === "date_desc" ||
+      (!params.sort && url === "events");
+
+    if (pageIndex > 0 && !isDateSort) {
+      return [
+        url,
+        { ...params, offset: pageIndex * API_LIMIT, limit: API_LIMIT },
+      ];
+    }
 
     if (pageIndex > 0 && previousPageData) {
       const lastDate = previousPageData[previousPageData.length - 1].start_time;
@@ -206,7 +218,8 @@ export default function Explore() {
         url,
         {
           ...params,
-          [isAscending ? "after" : "before"]: lastDate.toString(),
+          [params.sort === "date_asc" ? "after" : "before"]:
+            lastDate.toString(),
           limit: API_LIMIT,
         },
       ];
@@ -238,10 +251,17 @@ export default function Explore() {
     },
   });
 
-  const searchResults = useMemo(
-    () => (data ? ([] as SearchResult[]).concat(...data) : []),
-    [data],
-  );
+  // offset pages can overlap when results shift between page fetches
+  const searchResults = useMemo(() => {
+    if (!data) return [];
+
+    const seen = new Set<string>();
+    return data.flat().filter((result) => {
+      if (seen.has(result.id)) return false;
+      seen.add(result.id);
+      return true;
+    });
+  }, [data]);
   const isLoadingInitialData = !data && !isValidating;
   const isLoadingMore =
     isLoadingInitialData ||

web/src/types/search.ts

@@ -109,6 +109,7 @@ export type SearchQueryParams = {
   max_speed?: number;
   search_type?: string;
   limit?: number;
+  offset?: number;
   in_progress?: number;
   include_thumbnails?: number;
   query?: string;