feat: datatrails: include metric prefix filter (#81316)

* feat: datatrails: include metric prefix filter

* fix: remove current metric from related metrics list

* fix: Cascader issues

- handle empty items list when generating searchable options
- correct state management to ensure cascade shows cascade
  path of selection from search

* fix: remove custom value creation

packages/grafana-ui/src/components/Cascader/Cascader.tsx

@@ -93,7 +93,7 @@ export class Cascader extends PureComponent<CascaderProps, CascaderState> {
     for (const option of options) {
       const cpy = [...optionPath];
       cpy.push(option);
-      if (!option.items) {
+      if (!option.items || option.items.length === 0) {
         selectOptions.push({
           singleLabel: cpy[cpy.length - 1].label,
           label: cpy.map((o) => o.label).join(this.props.separator || DEFAULT_SEPARATOR),
@@ -135,23 +135,27 @@ export class Cascader extends PureComponent<CascaderProps, CascaderState> {
       : this.props.displayAllSelectedLevels
         ? selectedOptions.map((option) => option.label).join(this.props.separator || DEFAULT_SEPARATOR)
         : selectedOptions[selectedOptions.length - 1].label;
-    this.setState({
-      rcValue: value,
+    const state: CascaderState = {
+      rcValue: { value, label: activeLabel },
       focusCascade: true,
       activeLabel,
-    });
-
+      isSearching: false,
+    };
+    this.setState(state);
     this.props.onSelect(selectedOptions[selectedOptions.length - 1].value);
   };
 
   //For select
   onSelect = (obj: SelectableValue<string[]>) => {
     const valueArray = obj.value || [];
-    this.setState({
-      activeLabel: this.props.displayAllSelectedLevels ? obj.label : obj.singleLabel || '',
-      rcValue: valueArray,
+    const activeLabel = this.props.displayAllSelectedLevels ? obj.label : obj.singleLabel || '';
+    const state: CascaderState = {
+      activeLabel: activeLabel,
+      rcValue: { value: valueArray, label: activeLabel },
       isSearching: false,
-    });
+      focusCascade: false,
+    };
+    this.setState(state);
     this.props.onSelect(valueArray[valueArray.length - 1]);
   };
 

public/app/features/trails/MetricCategory/MetricCategoryCascader.tsx

@@ -0,0 +1,79 @@
+import React, { useMemo, useReducer, useState } from 'react';
+
+import { Cascader, CascaderOption, HorizontalGroup, Button } from '@grafana/ui';
+
+import { useMetricCategories } from './useMetricCategories';
+
+type Props = {
+  metricNames: string[];
+  onSelect: (prefix: string | undefined) => void;
+  disabled?: boolean;
+  initialValue?: string;
+};
+
+export function MetricCategoryCascader({ metricNames, onSelect, disabled, initialValue }: Props) {
+  const categoryTree = useMetricCategories(metricNames);
+  const options = useMemo(() => createCasaderOptions(categoryTree), [categoryTree]);
+
+  const [disableClear, setDisableClear] = useState(initialValue == null);
+
+  // Increments whenever clear is pressed, to reset the Cascader component
+  const [cascaderKey, resetCascader] = useReducer((x) => x + 1, 0);
+
+  const clear = () => {
+    resetCascader();
+    setDisableClear(true);
+    onSelect(undefined);
+  };
+
+  return (
+    <HorizontalGroup>
+      <Cascader
+        key={cascaderKey} // To reset the component to `undefined`
+        displayAllSelectedLevels={true}
+        width={40}
+        separator="_"
+        hideActiveLevelLabel={false}
+        placeholder={'No filter'}
+        onSelect={(prefix) => {
+          setDisableClear(!prefix);
+          onSelect(prefix);
+        }}
+        {...{ options, disabled, initialValue }}
+      />
+      <Button disabled={disableClear} onClick={clear} variant="secondary">
+        Clear
+      </Button>
+    </HorizontalGroup>
+  );
+}
+
+function createCasaderOptions(tree: ReturnType<typeof useMetricCategories>, currentPrefix = '') {
+  const categories = Object.entries(tree.children);
+
+  const options = categories.map(([metricPart, node]) => {
+    let subcategoryEntries = Object.entries(node.children);
+
+    while (subcategoryEntries.length === 1 && !node.isMetric) {
+      // There is only one subcategory, so we will join it with the current metricPart to reduce depth
+      const [subMetricPart, subNode] = subcategoryEntries[0];
+      metricPart = `${metricPart}_${subMetricPart}`;
+      // Extend the metric part name, because there is only one subcategory
+      node = subNode;
+      subcategoryEntries = Object.entries(node.children);
+    }
+
+    const value = currentPrefix + metricPart;
+    const subOptions = createCasaderOptions(node, value + '_');
+
+    const option: CascaderOption = {
+      value: value,
+      label: metricPart,
+      items: subOptions,
+    };
+
+    return option;
+  });
+
+  return options;
+}

public/app/features/trails/MetricCategory/useMetricCategories.ts

@@ -0,0 +1,43 @@
+import { useMemo } from 'react';
+
+export function useMetricCategories(metrics: string[]) {
+  return useMemo(() => processMetrics(metrics), [metrics]);
+}
+
+interface MetricPartNode {
+  isMetric?: boolean;
+  children: Record<string, MetricPartNode>;
+}
+
+function processMetrics(metrics: string[]) {
+  const categoryTree: MetricPartNode = { children: {} };
+
+  function insertMetric(metric: string) {
+    if (metric.indexOf(':') !== -1) {
+      // Ignore recording rules.
+      return;
+    }
+
+    const metricParts = metric.split('_');
+
+    let cursor = categoryTree;
+    for (const metricPart of metricParts) {
+      let node = cursor.children[metricPart];
+      if (!node) {
+        // Create new node
+        node = {
+          children: {},
+        };
+        // Insert it
+        cursor.children[metricPart] = node;
+      }
+      cursor = node;
+    }
+    // We know this node is a metric because it was for the last metricPart
+    cursor.isMetric = true;
+  }
+
+  metrics.forEach((metric) => insertMetric(metric));
+
+  return categoryTree;
+}

public/app/features/trails/MetricSelectScene.tsx

@@ -17,12 +17,13 @@ import {
   SceneCSSGridItem,
   SceneObjectRef,
   SceneQueryRunner,
-  VariableValueOption,
 } from '@grafana/scenes';
 import { VariableHide } from '@grafana/schema';
-import { Input, Text, useStyles2, InlineSwitch } from '@grafana/ui';
+import { Input, Text, useStyles2, InlineSwitch, Field, LoadingPlaceholder } from '@grafana/ui';
 
 import { getAutoQueriesForMetric } from './AutomaticMetricQueries/AutoQueryEngine';
+import { MetricCategoryCascader } from './MetricCategory/MetricCategoryCascader';
+import { MetricScene } from './MetricScene';
 import { SelectMetricAction } from './SelectMetricAction';
 import { hideEmptyPreviews } from './hideEmptyPreviews';
 import { getVariablesWithMetricConstant, trailDS, VAR_FILTERS_EXPR, VAR_METRIC_NAMES } from './shared';
@@ -42,6 +43,9 @@ export interface MetricSelectSceneState extends SceneObjectState {
   showHeading?: boolean;
   searchQuery?: string;
   showPreviews?: boolean;
+  prefixFilter?: string;
+  metricsAfterSearch?: string[];
+  metricsAfterFilter?: string[];
 }
 
 const ROW_PREVIEW_HEIGHT = '175px';
@@ -75,7 +79,7 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
   });
 
   private onVariableUpdateCompleted(): void {
-    this.updateMetrics();
+    this.updateMetrics(); // Entire pipeline must be performed
     this.buildLayout();
   }
 
@@ -103,33 +107,71 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
     });
   }
 
-  private updateMetrics() {
-    const trail = getTrailFor(this);
+  private getAllMetricNames() {
+    // Get the datasource metrics list from the VAR_METRIC_NAMES variable
     const variable = sceneGraph.lookupVariable(VAR_METRIC_NAMES, this);
 
     if (!(variable instanceof QueryVariable)) {
-      return;
+      return null;
     }
 
     if (variable.state.loading) {
+      return null;
+    }
+
+    const metricNames = variable.state.options.map((option) => option.value.toString());
+    return metricNames;
+  }
+
+  private applyMetricSearch() {
+    // This should only occur when the `searchQuery` changes, of if the `metricNames` change
+    const metricNames = this.getAllMetricNames();
+    if (metricNames == null) {
+      return;
+    }
+    const searchRegex = createSearchRegExp(this.state.searchQuery);
+
+    if (!searchRegex) {
+      this.setState({ metricsAfterSearch: metricNames });
+    } else {
+      const metricsAfterSearch = metricNames.filter((metric) => !searchRegex || searchRegex.test(metric));
+      this.setState({ metricsAfterSearch });
+    }
+  }
+
+  private applyMetricPrefixFilter() {
+    const { metricsAfterSearch, prefixFilter } = this.state;
+
+    if (!prefixFilter || !metricsAfterSearch) {
+      this.setState({ metricsAfterFilter: metricsAfterSearch });
+    } else {
+      const metricsAfterFilter = metricsAfterSearch.filter((metric) => metric.startsWith(prefixFilter));
+      this.setState({ metricsAfterFilter });
+    }
+  }
+
+  private updateMetrics(applySearchAndFilter = true) {
+    if (applySearchAndFilter) {
+      // Set to false if these are not required (because they can be assumed to have been suitably called).
+      this.applyMetricSearch();
+      this.applyMetricPrefixFilter();
+    }
+
+    const { metricsAfterFilter } = this.state;
+
+    if (!metricsAfterFilter) {
       return;
     }
 
-    const searchRegex = createSearchRegExp(this.state.searchQuery);
-    const metricNames = variable.state.options;
+    const metricNames = metricsAfterFilter;
+    const trail = getTrailFor(this);
     const sortedMetricNames =
       trail.state.metric !== undefined ? sortRelatedMetrics(metricNames, trail.state.metric) : metricNames;
     const metricsMap: Record<string, MetricPanel> = {};
     const metricsLimit = 120;
 
     for (let index = 0; index < sortedMetricNames.length; index++) {
-      const metric = sortedMetricNames[index];
-
-      const metricName = String(metric.value);
-
-      if (searchRegex && !searchRegex.test(metricName)) {
-        continue;
-      }
+      const metricName = sortedMetricNames[index];
 
       if (Object.keys(metricsMap).length > metricsLimit) {
         break;
@@ -138,6 +180,14 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
       metricsMap[metricName] = { name: metricName, index, loaded: false };
     }
 
+    try {
+      // If there is a current metric, do not present it
+      const currentMetric = sceneGraph.getAncestor(this, MetricScene).state.metric;
+      delete metricsMap[currentMetric];
+    } catch (err) {
+      // There is no current metric
+    }
+
     this.previewCache = metricsMap;
   }
 
@@ -207,7 +257,14 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
 
   public onSearchChange = (evt: React.SyntheticEvent<HTMLInputElement>) => {
     this.setState({ searchQuery: evt.currentTarget.value });
-    this.updateMetrics();
+    this.updateMetrics(); // Need to repeat entire pipeline
+    this.buildLayout();
+  };
+
+  public onPrefixFilterChange = (prefixFilter: string | undefined) => {
+    this.setState({ prefixFilter });
+    this.applyMetricPrefixFilter();
+    this.updateMetrics(false); // Only needed to applyMetricPrefixFilter
     this.buildLayout();
   };
 
@@ -217,13 +274,25 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
   };
 
   public static Component = ({ model }: SceneComponentProps<MetricSelectScene>) => {
-    const { showHeading, searchQuery, showPreviews, body } = model.useState();
+    const { showHeading, searchQuery, showPreviews, body, metricsAfterSearch, metricsAfterFilter, prefixFilter } =
+      model.useState();
     const { children } = body.useState();
     const styles = useStyles2(getStyles);
 
-    const searchTooStrictWarning = children.length === 0 && searchQuery && (
-      <div className={styles.alternateMessage}>There are no results found. Try adjusting your search or filters.</div>
-    );
+    const notLoaded = metricsAfterSearch === undefined && metricsAfterFilter === undefined && children.length === 0;
+
+    const tooStrict = children.length === 0 && (searchQuery || prefixFilter);
+
+    let status =
+      (notLoaded && <LoadingPlaceholder className={styles.statusMessage} text="Loading..." />) ||
+      (tooStrict && 'There are no results found. Try adjusting your search or filters.');
+
+    const showStatus = status && <div className={styles.statusMessage}>{status}</div>;
+
+    const prefixError =
+      prefixFilter && metricsAfterSearch != null && !metricsAfterFilter?.length
+        ? 'The current prefix filter is not available with the current search terms.'
+        : undefined;
 
     return (
       <div className={styles.container}>
@@ -236,7 +305,17 @@ export class MetricSelectScene extends SceneObjectBase<MetricSelectSceneState> {
           <Input placeholder="Search metrics" value={searchQuery} onChange={model.onSearchChange} />
           <InlineSwitch showLabel={true} label="Show previews" value={showPreviews} onChange={model.onTogglePreviews} />
         </div>
-        {searchTooStrictWarning}
+        <div className={styles.header}>
+          <Field label="Filter by prefix" error={prefixError} invalid={true}>
+            <MetricCategoryCascader
+              metricNames={metricsAfterSearch || []}
+              onSelect={model.onPrefixFilterChange}
+              disabled={metricsAfterSearch == null}
+              initialValue={prefixFilter}
+            />
+          </Field>
+        </div>
+        {showStatus}
         <model.state.body.Component model={model.state.body} />
       </div>
     );
@@ -291,13 +370,11 @@ function getCardPanelFor(metric: string) {
 }
 
 // Computes the Levenshtein distance between two strings, twice, once for the first half and once for the whole string.
-function sortRelatedMetrics(metricList: VariableValueOption[], metric: string) {
-  return metricList.sort((a, b) => {
-    const aValue = String(a.value);
+function sortRelatedMetrics(metricList: string[], metric: string) {
+  return metricList.sort((aValue, bValue) => {
     const aSplit = aValue.split('_');
     const aHalf = aSplit.slice(0, aSplit.length / 2).join('_');
 
-    const bValue = String(b.value);
     const bSplit = bValue.split('_');
     const bHalf = bSplit.slice(0, bSplit.length / 2).join('_');
 
@@ -324,7 +401,7 @@ function getStyles(theme: GrafanaTheme2) {
       gap: theme.spacing(2),
       marginBottom: theme.spacing(1),
     }),
-    alternateMessage: css({
+    statusMessage: css({
       fontStyle: 'italic',
       marginTop: theme.spacing(7),
       textAlign: 'center',