Improve Explore SQL query memory usage (#14451)

* Remove sql window function in explore endpoint

* don't revalidate first page on every fetch

frigate/api/event.py

@@ -259,66 +259,61 @@ def events(params: EventsQueryParams = Depends()):
 
 @router.get("/events/explore")
 def events_explore(limit: int = 10):
-    subquery = Event.select(
+    # get distinct labels for all events
-        Event.id,
+    distinct_labels = Event.select(Event.label).distinct().order_by(Event.label)
-        Event.camera,
-        Event.label,
-        Event.zones,
-        Event.start_time,
-        Event.end_time,
-        Event.has_clip,
-        Event.has_snapshot,
-        Event.plus_id,
-        Event.retain_indefinitely,
-        Event.sub_label,
-        Event.top_score,
-        Event.false_positive,
-        Event.box,
-        Event.data,
-        fn.rank()
-        .over(partition_by=[Event.label], order_by=[Event.start_time.desc()])
-        .alias("rank"),
-        fn.COUNT(Event.id).over(partition_by=[Event.label]).alias("event_count"),
-    ).alias("subquery")
 
-    query = (
+    label_counts = {}
-        Event.select(
-            subquery.c.id,
-            subquery.c.camera,
-            subquery.c.label,
-            subquery.c.zones,
-            subquery.c.start_time,
-            subquery.c.end_time,
-            subquery.c.has_clip,
-            subquery.c.has_snapshot,
-            subquery.c.plus_id,
-            subquery.c.retain_indefinitely,
-            subquery.c.sub_label,
-            subquery.c.top_score,
-            subquery.c.false_positive,
-            subquery.c.box,
-            subquery.c.data,
-            subquery.c.event_count,
-        )
-        .from_(subquery)
-        .where(subquery.c.rank <= limit)
-        .order_by(subquery.c.event_count.desc(), subquery.c.start_time.desc())
-        .dicts()
-    )
 
-    events = list(query.iterator())
+    def event_generator():
+        for label_obj in distinct_labels.iterator():
+            label = label_obj.label
 
-    processed_events = [
+            # get most recent events for this label
-        {k: v for k, v in event.items() if k != "data"}
+            label_events = (
-        | {
+                Event.select()
-            "data": {
+                .where(Event.label == label)
-                k: v
+                .order_by(Event.start_time.desc())
-                for k, v in event["data"].items()
+                .limit(limit)
-                if k in ["type", "score", "top_score", "description"]
+                .iterator()
+            )
+
+            # count total events for this label
+            label_counts[label] = Event.select().where(Event.label == label).count()
+
+            yield from label_events
+
+    def process_events():
+        for event in event_generator():
+            processed_event = {
+                "id": event.id,
+                "camera": event.camera,
+                "label": event.label,
+                "zones": event.zones,
+                "start_time": event.start_time,
+                "end_time": event.end_time,
+                "has_clip": event.has_clip,
+                "has_snapshot": event.has_snapshot,
+                "plus_id": event.plus_id,
+                "retain_indefinitely": event.retain_indefinitely,
+                "sub_label": event.sub_label,
+                "top_score": event.top_score,
+                "false_positive": event.false_positive,
+                "box": event.box,
+                "data": {
+                    k: v
+                    for k, v in event.data.items()
+                    if k in ["type", "score", "top_score", "description"]
+                },
+                "event_count": label_counts[event.label],
             }
-        }
+            yield processed_event
-        for event in events
+
-    ]
+    # convert iterator to list and sort
+    processed_events = sorted(
+        process_events(),
+        key=lambda x: (x["event_count"], x["start_time"]),
+        reverse=True,
+    )
 
     return JSONResponse(content=processed_events)
 

web/src/pages/Explore.tsx

@@ -177,7 +177,7 @@ export default function Explore() {
   const { data, size, setSize, isValidating, mutate } = useSWRInfinite<
     SearchResult[]
   >(getKey, {
-    revalidateFirstPage: true,
+    revalidateFirstPage: false,
     revalidateOnFocus: true,
     revalidateAll: false,
   });