Data trails: Sort related metrics and hide empty panels (#79397)

* Use levenshtein method to sort metric names

* Optionally hide empty panels in MetricSelectScene

* Transform ignore leven

* Refactor code to use $behaviours. Move preview cache to class variable instead of state

* Use lazy loading for metric scene

* Update scenes lib

* simplify behavior

* Remove hide empty toggle

* Bump scenes

---------

Co-authored-by: Torkel Ödegaard <torkel@grafana.com>

jest.config.js

@@ -3,7 +3,16 @@
 // 2. Any wrong timezone handling could be hidden if we use UTC/GMT local time (which would happen in CI).
 process.env.TZ = 'Pacific/Easter'; // UTC-06:00 or UTC-05:00 depending on daylight savings
 
-const esModules = ['ol', 'd3', 'd3-color', 'd3-interpolate', 'delaunator', 'internmap', 'robust-predicates'].join('|');
+const esModules = [
+  'ol',
+  'd3',
+  'd3-color',
+  'd3-interpolate',
+  'delaunator',
+  'internmap',
+  'robust-predicates',
+  'leven',
+].join('|');
 
 module.exports = {
   verbose: false,

package.json

@@ -255,7 +255,7 @@
     "@grafana/lezer-traceql": "0.0.12",
     "@grafana/monaco-logql": "^0.0.7",
     "@grafana/runtime": "workspace:*",
-    "@grafana/scenes": "1.28.0",
+    "@grafana/scenes": "1.28.5",
     "@grafana/schema": "workspace:*",
     "@grafana/ui": "workspace:*",
     "@kusto/monaco-kusto": "^7.4.0",
@@ -335,6 +335,7 @@
     "json-source-map": "0.6.1",
     "jsurl": "^0.1.5",
     "kbar": "0.1.0-beta.44",
+    "leven": "^4.0.0",
     "lodash": "4.17.21",
     "logfmt": "^1.3.2",
     "lru-cache": "10.0.0",

public/app/features/trails/AutomaticMetricQueries/AutoQueryEngine.ts

@@ -1,8 +1,8 @@
-import { PanelBuilders, SceneQueryRunner, VizPanelBuilder } from '@grafana/scenes';
+import { PanelBuilders, VizPanelBuilder } from '@grafana/scenes';
 import { PromQuery } from 'app/plugins/datasource/prometheus/types';
 import { HeatmapColorMode } from 'app/plugins/panel/heatmap/types';
 
-import { KEY_SQR_METRIC_VIZ_QUERY, trailDS, VAR_FILTERS_EXPR, VAR_GROUP_BY_EXP, VAR_METRIC_EXPR } from '../shared';
+import { VAR_FILTERS_EXPR, VAR_GROUP_BY_EXP, VAR_METRIC_EXPR } from '../shared';
 
 export interface AutoQueryDef {
   variant: string;
@@ -154,30 +154,13 @@ function getQueriesForBucketMetric(metric: string): AutoQueryInfo {
 function simpleGraphBuilder(def: AutoQueryDef) {
   return PanelBuilders.timeseries()
     .setTitle(def.title)
-    .setData(
-      new SceneQueryRunner({
-        datasource: trailDS,
-        maxDataPoints: 200,
-        queries: def.queries,
-      })
-    )
     .setUnit(def.unit)
     .setOption('legend', { showLegend: false })
     .setCustomFieldConfig('fillOpacity', 9);
 }
 
 function percentilesGraphBuilder(def: AutoQueryDef) {
-  return PanelBuilders.timeseries()
+  return PanelBuilders.timeseries().setTitle(def.title).setUnit(def.unit).setCustomFieldConfig('fillOpacity', 9);
-    .setTitle(def.title)
-    .setData(
-      new SceneQueryRunner({
-        datasource: trailDS,
-        maxDataPoints: 200,
-        queries: def.queries,
-      })
-    )
-    .setUnit(def.unit)
-    .setCustomFieldConfig('fillOpacity', 9);
 }
 
 function heatmapGraphBuilder(def: AutoQueryDef) {
@@ -191,12 +174,5 @@ function heatmapGraphBuilder(def: AutoQueryDef) {
       scheme: 'Spectral',
       steps: 32,
       reverse: false,
-    })
+    });
-    .setData(
-      new SceneQueryRunner({
-        key: KEY_SQR_METRIC_VIZ_QUERY,
-        datasource: trailDS,
-        queries: def.queries,
-      })
-    );
 }

public/app/features/trails/AutomaticMetricQueries/AutoVizPanel.tsx

@@ -2,9 +2,10 @@ import { css } from '@emotion/css';
 import React from 'react';
 
 import { GrafanaTheme2 } from '@grafana/data';
-import { SceneObjectState, SceneObjectBase, SceneComponentProps, VizPanel } from '@grafana/scenes';
+import { SceneObjectState, SceneObjectBase, SceneComponentProps, VizPanel, SceneQueryRunner } from '@grafana/scenes';
 import { Field, RadioButtonGroup, useStyles2, Stack } from '@grafana/ui';
 
+import { trailDS } from '../shared';
 import { getTrailSettings } from '../utils';
 
 import { AutoQueryDef, AutoQueryInfo } from './AutoQueryEngine';
@@ -49,7 +50,17 @@ export class AutoVizPanel extends SceneObjectBase<AutoVizPanelState> {
   };
 
   private getVizPanelFor(def: AutoQueryDef) {
-    return def.vizBuilder(def).setHeaderActions(this.getQuerySelector(def)).build();
+    return def
+      .vizBuilder(def)
+      .setData(
+        new SceneQueryRunner({
+          datasource: trailDS,
+          maxDataPoints: 500,
+          queries: def.queries,
+        })
+      )
+      .setHeaderActions(this.getQuerySelector(def))
+      .build();
   }
 
   public static Component = ({ model }: SceneComponentProps<AutoVizPanel>) => {

public/app/features/trails/MetricSelectScene.tsx

@@ -1,4 +1,5 @@
 import { css } from '@emotion/css';
+import leven from 'leven';
 import React from 'react';
 
 import { GrafanaTheme2 } from '@grafana/data';
@@ -15,14 +16,27 @@ import {
   SceneVariable,
   SceneCSSGridLayout,
   SceneCSSGridItem,
+  SceneObjectRef,
+  SceneQueryRunner,
+  VariableValueOption,
 } from '@grafana/scenes';
 import { VariableHide } from '@grafana/schema';
 import { Input, Text, useStyles2, InlineSwitch } from '@grafana/ui';
 
 import { getAutoQueriesForMetric } from './AutomaticMetricQueries/AutoQueryEngine';
 import { SelectMetricAction } from './SelectMetricAction';
+import { hideEmptyPreviews } from './hideEmptyPreviews';
 import { getVariablesWithMetricConstant, trailDS, VAR_FILTERS_EXPR, VAR_METRIC_NAMES } from './shared';
-import { getColorByIndex } from './utils';
+import { getColorByIndex, getTrailFor } from './utils';
+
+interface MetricPanel {
+  name: string;
+  index: number;
+  itemRef?: SceneObjectRef<SceneCSSGridItem>;
+  isEmpty?: boolean;
+  isPanel?: boolean;
+  loaded?: boolean;
+}
 
 export interface MetricSelectSceneState extends SceneObjectState {
   body: SceneCSSGridLayout;
@@ -35,6 +49,8 @@ const ROW_PREVIEW_HEIGHT = '175px';
 const ROW_CARD_HEIGHT = '64px';
 
 export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
+  private previewCache: Record<string, MetricPanel> = {};
+
   constructor(state: Partial<MetricSelectSceneState>) {
     super({
       $variables: state.$variables ?? getMetricNamesVariableSet(),
@@ -44,6 +60,7 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
           children: [],
           templateColumns: 'repeat(auto-fill, minmax(450px, 1fr))',
           autoRows: ROW_PREVIEW_HEIGHT,
+          isLazy: true,
         }),
       showPreviews: true,
       ...state,
@@ -59,6 +76,7 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
 
   private _onVariableChanged(changedVariables: Set<SceneVariable>, dependencyChanged: boolean): void {
     if (dependencyChanged) {
+      this.updateMetrics();
       this.buildLayout();
     }
   }
@@ -73,6 +91,58 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
     }
   }
 
+  private sortedPreviewMetrics() {
+    return Object.values(this.previewCache).sort((a, b) => {
+      if (a.isEmpty && b.isEmpty) {
+        return a.index - b.index;
+      }
+      if (a.isEmpty) {
+        return 1;
+      }
+      if (b.isEmpty) {
+        return -1;
+      }
+      return a.index - b.index;
+    });
+  }
+
+  private updateMetrics() {
+    const trail = getTrailFor(this);
+    const variable = sceneGraph.lookupVariable(VAR_METRIC_NAMES, this);
+
+    if (!(variable instanceof QueryVariable)) {
+      return;
+    }
+
+    if (variable.state.loading) {
+      return;
+    }
+
+    const searchRegex = new RegExp(this.state.searchQuery ?? '.*');
+    const metricNames = variable.state.options;
+    const sortedMetricNames =
+      trail.state.metric !== undefined ? sortRelatedMetrics(metricNames, trail.state.metric) : metricNames;
+    const metricsMap: Record<string, MetricPanel> = {};
+    const metricsLimit = 120;
+
+    for (let index = 0; index < sortedMetricNames.length; index++) {
+      const metric = sortedMetricNames[index];
+
+      const metricName = String(metric.value);
+      if (!metricName.match(searchRegex)) {
+        continue;
+      }
+
+      if (Object.keys(metricsMap).length > metricsLimit) {
+        break;
+      }
+
+      metricsMap[metricName] = { name: metricName, index, loaded: false };
+    }
+
+    this.previewCache = metricsMap;
+  }
+
   private buildLayout() {
     // Temp hack when going back to select metric scene and variable updates
     if (this.ignoreNextUpdate) {
@@ -90,39 +160,33 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
       return;
     }
 
-    const searchRegex = new RegExp(this.state.searchQuery ?? '.*');
+    if (!Object.keys(this.previewCache).length) {
-    const metricNames = variable.state.options;
+      this.updateMetrics();
+    }
+
     const children: SceneFlexItem[] = [];
-    const showPreviews = this.state.showPreviews;
-    const previewLimit = 20;
-    const cardLimit = 50;
 
-    for (let index = 0; index < metricNames.length; index++) {
+    const metricsList = this.sortedPreviewMetrics();
-      const metric = metricNames[index];
+    for (let index = 0; index < metricsList.length; index++) {
+      const metric = metricsList[index];
 
-      const metricName = String(metric.value);
+      if (metric.itemRef && metric.isPanel) {
-      if (!metricName.match(searchRegex)) {
+        children.push(metric.itemRef.resolve());
         continue;
       }
-
+      if (this.state.showPreviews) {
-      if (children.length > cardLimit) {
+        const panel = getPreviewPanelFor(metric.name, index);
-        break;
+        metric.itemRef = panel.getRef();
-      }
+        metric.isPanel = true;
-
+        children.push(panel);
-      if (showPreviews && children.length < previewLimit) {
-        children.push(
-          new SceneCSSGridItem({
-            $variables: getVariablesWithMetricConstant(metricName),
-            body: getPreviewPanelFor(metricName, index),
-          })
-        );
       } else {
-        children.push(
+        const panel = new SceneCSSGridItem({
-          new SceneCSSGridItem({
+          $variables: getVariablesWithMetricConstant(metric.name),
-            $variables: getVariablesWithMetricConstant(metricName),
+          body: getCardPanelFor(metric.name),
-            body: getCardPanelFor(metricName),
+        });
-          })
+        metric.itemRef = panel.getRef();
-        );
+        metric.isPanel = false;
+        children.push(panel);
       }
     }
 
@@ -131,8 +195,19 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
     this.state.body.setState({ children, autoRows: rowTemplate });
   }
 
+  public updateMetricPanel = (metric: string, isLoaded?: boolean, isEmpty?: boolean) => {
+    const metricPanel = this.previewCache[metric];
+    if (metricPanel) {
+      metricPanel.isEmpty = isEmpty;
+      metricPanel.loaded = isLoaded;
+      this.previewCache[metric] = metricPanel;
+      this.buildLayout();
+    }
+  };
+
   public onSearchChange = (evt: React.SyntheticEvent<HTMLInputElement>) => {
     this.setState({ searchQuery: evt.currentTarget.value });
+    this.updateMetrics();
     this.buildLayout();
   };
 
@@ -181,11 +256,22 @@ function getMetricNamesVariableSet() {
 function getPreviewPanelFor(metric: string, index: number) {
   const autoQuery = getAutoQueriesForMetric(metric);
 
-  return autoQuery.preview
+  const vizPanel = autoQuery.preview
     .vizBuilder(autoQuery.preview)
     .setColor({ mode: 'fixed', fixedColor: getColorByIndex(index) })
     .setHeaderActions(new SelectMetricAction({ metric, title: 'Select' }))
     .build();
+
+  return new SceneCSSGridItem({
+    $variables: getVariablesWithMetricConstant(metric),
+    $behaviors: [hideEmptyPreviews(metric)],
+    $data: new SceneQueryRunner({
+      datasource: trailDS,
+      maxDataPoints: 200,
+      queries: autoQuery.preview.queries,
+    }),
+    body: vizPanel,
+  });
 }
 
 function getCardPanelFor(metric: string) {
@@ -196,6 +282,24 @@ function getCardPanelFor(metric: string) {
     .build();
 }
 
+// Computes the Levenshtein distance between two strings, twice, once for the first half and once for the whole string.
+function sortRelatedMetrics(metricList: VariableValueOption[], metric: string) {
+  return metricList.sort((a, b) => {
+    const aValue = String(a.value);
+    const aSplit = aValue.split('_');
+    const aHalf = aSplit.slice(0, aSplit.length / 2).join('_');
+
+    const bValue = String(b.value);
+    const bSplit = bValue.split('_');
+    const bHalf = bSplit.slice(0, bSplit.length / 2).join('_');
+
+    return (
+      (leven(aHalf, metric!) || 0 + (leven(aValue, metric!) || 0)) -
+      (leven(bHalf, metric!) || 0 + (leven(bValue, metric!) || 0))
+    );
+  });
+}
+
 function getStyles(theme: GrafanaTheme2) {
   return {
     container: css({

public/app/features/trails/hideEmptyPreviews.ts

@@ -0,0 +1,43 @@
+import { FieldType, LoadingState } from '@grafana/data';
+import { SceneCSSGridItem, sceneGraph } from '@grafana/scenes';
+
+import { MetricSelectScene } from './MetricSelectScene';
+
+export function hideEmptyPreviews(metric: string) {
+  return (gridItem: SceneCSSGridItem) => {
+    const data = sceneGraph.getData(gridItem);
+    if (!data) {
+      return;
+    }
+
+    data.subscribeToState((state) => {
+      if (state.data?.state === LoadingState.Loading) {
+        return;
+      }
+      const scene = sceneGraph.getAncestor(gridItem, MetricSelectScene);
+
+      if (!state.data?.series.length) {
+        scene.updateMetricPanel(metric, true, true);
+        return;
+      }
+
+      let hasValue = false;
+      for (const frame of state.data.series) {
+        for (const field of frame.fields) {
+          if (field.type !== FieldType.number) {
+            continue;
+          }
+
+          hasValue = field.values.some((v) => v != null && !isNaN(v) && v !== 0);
+          if (hasValue) {
+            break;
+          }
+        }
+        if (hasValue) {
+          break;
+        }
+      }
+      scene.updateMetricPanel(metric, true, !hasValue);
+    });
+  };
+}

yarn.lock

@@ -3290,9 +3290,9 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@grafana/scenes@npm:1.28.0":
+"@grafana/scenes@npm:1.28.5":
-  version: 1.28.0
+  version: 1.28.5
-  resolution: "@grafana/scenes@npm:1.28.0"
+  resolution: "@grafana/scenes@npm:1.28.5"
   dependencies:
     "@grafana/e2e-selectors": "npm:10.0.2"
     react-grid-layout: "npm:1.3.4"
@@ -3304,7 +3304,7 @@ __metadata:
     "@grafana/runtime": 10.0.3
     "@grafana/schema": 10.0.3
     "@grafana/ui": 10.0.3
-  checksum: 0973206c4485cad15ceb41f031e96e0f1f075be24570f527bbcb17dd56d5cd362385c04acef8f7aa240c3bb8b045d2270fab2dbb2f18e7e2850ab67a13a3d268
+  checksum: 9aec680a56196f844908afb395a2c401d85333ebe4f20cf1cdfdbd4a675d22174b647bb63155f95228198e6ba7a431392b8a27bcea44072ccdfa04c95e41dacb
   languageName: node
   linkType: hard
 
@@ -17317,7 +17317,7 @@ __metadata:
     "@grafana/lezer-traceql": "npm:0.0.12"
     "@grafana/monaco-logql": "npm:^0.0.7"
     "@grafana/runtime": "workspace:*"
-    "@grafana/scenes": "npm:1.28.0"
+    "@grafana/scenes": "npm:1.28.5"
     "@grafana/schema": "workspace:*"
     "@grafana/tsconfig": "npm:^1.3.0-rc1"
     "@grafana/ui": "workspace:*"
@@ -17510,6 +17510,7 @@ __metadata:
     jsurl: "npm:^0.1.5"
     kbar: "npm:0.1.0-beta.44"
     lerna: "npm:7.4.1"
+    leven: "npm:^4.0.0"
     lodash: "npm:4.17.21"
     logfmt: "npm:^1.3.2"
     lru-cache: "npm:10.0.0"
@@ -21144,6 +21145,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"leven@npm:^4.0.0":
+  version: 4.0.0
+  resolution: "leven@npm:4.0.0"
+  checksum: d70b9fef4cca487a38021bb173a5cae98d39b1c7f4a5b2439763bd89df8e389f178a3c941b6fc3fab1582f5052b5e8c91353d9607799a2ad3841e7ea22f9720f
+  languageName: node
+  linkType: hard
+
 "levn@npm:^0.4.1":
   version: 0.4.1
   resolution: "levn@npm:0.4.1"