TransformationFilter: Include transformation outputs in transformation filtering options (#98323)

* wip: include transformation output as filtering option * add refId to joinByField transformation * clean up * add refId to transformations that create new data frames * adjust duplicate query removal for filtering options * refactor transformation input/output subscription effect * adjust input data frame filtering logic to include transformations as input for debug view * transformation filter can only filter on output of previous transformation
2025-02-25 18:55:37 -06:00 · 2025-01-16 14:45:23 +00:00 · 2025-01-16 14:45:23 +00:00 · a32eed1d13
commit a32eed1d13
parent 3df1fa86ae
14 changed files with 113 additions and 71 deletions
--- a/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts
+++ b/packages/grafana-data/src/transformations/transformers/ensureColumns.test.ts
@ -42,7 +42,10 @@ describe('ensureColumns transformer', () => {
      options: {},
    };

-    const data = [seriesA, seriesBC];
+    const data = [
+      { refId: 'A', ...seriesA },
+      { refId: 'B', ...seriesBC },
+    ];

    await expect(transformDataFrame([cfg], data)).toEmitValuesWith((received) => {
      const filtered = received[0];
@ -109,6 +112,7 @@ describe('ensureColumns transformer', () => {
            },
          ],
          "length": 2,
+          "refId": "joinByField-A-B",
        }
      `);
    });
--- a/packages/grafana-data/src/transformations/transformers/histogram.ts
+++ b/packages/grafana-data/src/transformations/transformers/histogram.ts
@ -592,5 +592,6 @@ export function histogramFieldsToFrame(info: HistogramFields, theme?: GrafanaThe
      type: DataFrameType.Histogram,
    },
    fields: [info.xMin, info.xMax, ...info.counts],
+    refId: `${DataTransformerID.histogram}`,
  };
 }
--- a/packages/grafana-data/src/transformations/transformers/joinByField.ts
+++ b/packages/grafana-data/src/transformations/transformers/joinByField.ts
@ -42,6 +42,7 @@ export const joinByFieldTransformer: SynchronousDataTransformerInfo<JoinByFieldO
        }
        const joined = joinDataFrames({ frames: data, joinBy, mode: options.mode });
        if (joined) {
+          joined.refId = `${DataTransformerID.joinByField}-${data.map((frame) => frame.refId).join('-')}`;
          return [joined];
        }
      }
--- a/packages/grafana-data/src/transformations/transformers/merge.ts
+++ b/packages/grafana-data/src/transformations/transformers/merge.ts
@ -43,7 +43,10 @@ export const mergeTransformer: DataTransformerInfo<MergeTransformerOptions> = {
        const fieldNames = new Set<string>();
        const fieldIndexByName: Record<string, Record<number, number>> = {};
        const fieldNamesForKey: string[] = [];
-        const dataFrame = new MutableDataFrame();
+        const dataFrame = new MutableDataFrame({
+          refId: `${DataTransformerID.merge}-${data.map((frame) => frame.refId).join('-')}`,
+          fields: [],
+        });

        for (let frameIndex = 0; frameIndex < data.length; frameIndex++) {
          const frame = data[frameIndex];
--- a/packages/grafana-data/src/transformations/transformers/reduce.ts
+++ b/packages/grafana-data/src/transformations/transformers/reduce.ts
@ -56,7 +56,9 @@ export const reduceTransformer: DataTransformerInfo<ReduceTransformerOptions> =

        // Add a row for each series
        const res = reduceSeriesToRows(data, matcher, options.reducers, options.labelsToFields);
-        return res ? [res] : [];
+        return res
+          ? [{ ...res, refId: `${DataTransformerID.reduce}-${data.map((frame) => frame.refId).join('-')}` }]
+          : [];
      })
    ),
 };
--- a/packages/grafana-data/src/transformations/transformers/seriesToRows.ts
+++ b/packages/grafana-data/src/transformations/transformers/seriesToRows.ts
@ -37,7 +37,10 @@ export const seriesToRowsTransformer: DataTransformerInfo<SeriesToRowsTransforme

        const timeFieldByIndex: Record<number, number> = {};
        const targetFields = new Set<string>();
-        const dataFrame = new MutableDataFrame();
+        const dataFrame = new MutableDataFrame({
+          refId: `${DataTransformerID.seriesToRows}-${data.map((frame) => frame.refId).join('-')}`,
+          fields: [],
+        });
        const metricField: Field = {
          name: TIME_SERIES_METRIC_FIELD_NAME,
          values: [],
--- a/packages/grafana-data/src/transformations/transformers/transpose.ts
+++ b/packages/grafana-data/src/transformations/transformers/transpose.ts
@ -80,6 +80,7 @@ function transposeDataFrame(options: TransposeTransformerOptions, data: DataFram
      ...frame,
      fields: newFields,
      length: Math.max(...newFields.map((field) => field.values.length)),
+      refId: `${DataTransformerID.transpose}-${frame.refId}`,
    };
  });
 }
--- a/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx
+++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx
@ -1,26 +1,17 @@
 import { css } from '@emotion/css';
-import { createElement, useEffect, useMemo, useState } from 'react';
-import { mergeMap } from 'rxjs/operators';
+import { createElement, useMemo } from 'react';

-import {
-  DataFrame,
-  DataTransformerConfig,
-  GrafanaTheme2,
-  transformDataFrame,
-  TransformerRegistryItem,
-  getFrameMatchers,
-  DataTransformContext,
-} from '@grafana/data';
+import { DataFrame, DataTransformerConfig, GrafanaTheme2, TransformerRegistryItem } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { getTemplateSrv } from '@grafana/runtime';
 import { Icon, JSONFormatter, useStyles2, Drawer } from '@grafana/ui';

 import { TransformationsEditorTransformation } from './types';

 interface TransformationEditorProps {
+  input: DataFrame[];
+  output: DataFrame[];
  debugMode?: boolean;
  index: number;
-  data: DataFrame[];
  uiConfig: TransformerRegistryItem;
  configs: TransformationsEditorTransformation[];
  onChange: (index: number, config: DataTransformerConfig) => void;
@ -28,45 +19,18 @@ interface TransformationEditorProps {
 }

 export const TransformationEditor = ({
+  input,
+  output,
  debugMode,
  index,
-  data,
  uiConfig,
  configs,
  onChange,
  toggleShowDebug,
 }: TransformationEditorProps) => {
  const styles = useStyles2(getStyles);
-  const [input, setInput] = useState<DataFrame[]>([]);
-  const [output, setOutput] = useState<DataFrame[]>([]);
  const config = useMemo(() => configs[index], [configs, index]);

-  useEffect(() => {
-    const config = configs[index].transformation;
-    const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
-    const inputTransforms = configs.slice(0, index).map((t) => t.transformation);
-    const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation);
-
-    const ctx: DataTransformContext = {
-      interpolate: (v: string) => getTemplateSrv().replace(v),
-    };
-
-    const inputSubscription = transformDataFrame(inputTransforms, data, ctx).subscribe((v) => {
-      if (matcher) {
-        v = data.filter((v) => matcher(v));
-      }
-      setInput(v);
-    });
-    const outputSubscription = transformDataFrame(inputTransforms, data, ctx)
-      .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before, ctx)))
-      .subscribe(setOutput);
-
-    return function unsubscribe() {
-      inputSubscription.unsubscribe();
-      outputSubscription.unsubscribe();
-    };
-  }, [index, data, configs]);
-
  const editor = useMemo(
    () =>
      createElement(uiConfig.editor, {
--- a/public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx
+++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx
@ -1,40 +1,35 @@
 import { css } from '@emotion/css';
 import { useMemo } from 'react';

-import {
-  DataTransformerConfig,
-  GrafanaTheme2,
-  StandardEditorContext,
-  StandardEditorsRegistryItem,
-} from '@grafana/data';
+import { DataFrame, DataTransformerConfig, GrafanaTheme2 } from '@grafana/data';
 import { DataTopic } from '@grafana/schema';
 import { Field, Select, useStyles2 } from '@grafana/ui';
 import { FrameMultiSelectionEditor } from 'app/plugins/panel/geomap/editor/FrameSelectionEditor';

-import { TransformationData } from './TransformationsEditor';
-
 interface TransformationFilterProps {
+  /** data frames from the output of previous transformation */
+  data: DataFrame[];
  index: number;
  config: DataTransformerConfig;
-  data: TransformationData;
+  annotations?: DataFrame[];
  onChange: (index: number, config: DataTransformerConfig) => void;
 }

-export const TransformationFilter = ({ index, data, config, onChange }: TransformationFilterProps) => {
+export const TransformationFilter = ({ index, annotations, config, onChange, data }: TransformationFilterProps) => {
  const styles = useStyles2(getStyles);

  const opts = useMemo(() => {
    return {
      // eslint-disable-next-line
-      context: { data: data.series } as StandardEditorContext<unknown>,
-      showTopic: true || data.annotations?.length || config.topic?.length,
+      context: { data },
+      showTopic: true || annotations?.length || config.topic?.length,
      showFilter: config.topic !== DataTopic.Annotations,
      source: [
-        { value: DataTopic.Series, label: `Query results` },
+        { value: DataTopic.Series, label: `Query and Transformation results` },
        { value: DataTopic.Annotations, label: `Annotation data` },
      ],
    };
-  }, [data, config.topic]);
+  }, [data, annotations?.length, config.topic]);

  return (
    <div className={styles.wrapper}>
@ -59,8 +54,6 @@ export const TransformationFilter = ({ index, data, config, onChange }: Transfor
            <FrameMultiSelectionEditor
              value={config.filter!}
              context={opts.context}
-              // eslint-disable-next-line
-              item={{} as StandardEditorsRegistryItem}
              onChange={(filter) => onChange(index, { ...config, filter })}
            />
          )}
--- a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx
+++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx
@ -1,10 +1,18 @@
-import { useCallback } from 'react';
-import * as React from 'react';
+import { useCallback, useEffect, useState } from 'react';
 import { useToggle } from 'react-use';
+import { mergeMap } from 'rxjs';

-import { DataTransformerConfig, TransformerRegistryItem, FrameMatcherID, DataTopic } from '@grafana/data';
+import {
+  DataTransformerConfig,
+  TransformerRegistryItem,
+  FrameMatcherID,
+  DataTransformContext,
+  getFrameMatchers,
+  transformDataFrame,
+  DataFrame,
+} from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
-import { reportInteraction } from '@grafana/runtime';
+import { getTemplateSrv, reportInteraction } from '@grafana/runtime';
 import { ConfirmModal } from '@grafana/ui';
 import {
  QueryOperationAction,
@ -46,6 +54,10 @@ export const TransformationOperationRow = ({
  const topic = configs[index].transformation.topic;
  const showFilterEditor = configs[index].transformation.filter != null || topic != null;
  const showFilterToggle = showFilterEditor || data.series.length > 0 || (data.annotations?.length ?? 0) > 0;
+  const [input, setInput] = useState<DataFrame[]>([]);
+  const [output, setOutput] = useState<DataFrame[]>([]);
+  // output of previous transformation
+  const [prevOutput, setPrevOutput] = useState<DataFrame[]>([]);

  const onDisableToggle = useCallback(
    (index: number) => {
@ -92,6 +104,48 @@ export const TransformationOperationRow = ({
    [configs, index]
  );

+  useEffect(() => {
+    const config = configs[index].transformation;
+    const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
+    // we need previous transformation index to get its outputs
+    //    to be used in this transforms inputs
+    const prevTransformIndex = index - 1;
+
+    let prevInputTransforms: Array<DataTransformerConfig<{}>> = [];
+    let prevOutputTransforms: Array<DataTransformerConfig<{}>> = [];
+
+    if (prevTransformIndex >= 0) {
+      prevInputTransforms = configs.slice(0, prevTransformIndex).map((t) => t.transformation);
+      prevOutputTransforms = configs.slice(prevTransformIndex, index).map((t) => t.transformation);
+    }
+
+    const inputTransforms = configs.slice(0, index).map((t) => t.transformation);
+    const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation);
+
+    const ctx: DataTransformContext = {
+      interpolate: (v: string) => getTemplateSrv().replace(v),
+    };
+
+    const inputSubscription = transformDataFrame(inputTransforms, data.series, ctx).subscribe((data) => {
+      if (matcher) {
+        data = data.filter((frame) => matcher(frame));
+      }
+      setInput(data);
+    });
+    const outputSubscription = transformDataFrame(inputTransforms, data.series, ctx)
+      .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before, ctx)))
+      .subscribe(setOutput);
+    const prevOutputSubscription = transformDataFrame(prevInputTransforms, data.series, ctx)
+      .pipe(mergeMap((before) => transformDataFrame(prevOutputTransforms, before, ctx)))
+      .subscribe(setPrevOutput);
+
+    return function unsubscribe() {
+      inputSubscription.unsubscribe();
+      outputSubscription.unsubscribe();
+      prevOutputSubscription.unsubscribe();
+    };
+  }, [index, data, configs]);
+
  const renderActions = () => {
    return (
      <>
@ -162,13 +216,20 @@ export const TransformationOperationRow = ({
        }}
      >
        {showFilterEditor && (
-          <TransformationFilter index={index} config={configs[index].transformation} data={data} onChange={onChange} />
+          <TransformationFilter
+            data={prevOutput}
+            index={index}
+            config={configs[index].transformation}
+            annotations={data.annotations}
+            onChange={onChange}
+          />
        )}

        <TransformationEditor
+          input={input}
+          output={output}
          debugMode={showDebug}
          index={index}
-          data={topic === DataTopic.Annotations ? (data.annotations ?? []) : data.series}
          configs={configs}
          uiConfig={uiConfig}
          onChange={onChange}
--- a/public/app/features/transformers/joinByLabels/joinByLabels.ts
+++ b/public/app/features/transformers/joinByLabels/joinByLabels.ts
@ -35,7 +35,7 @@ interface JoinValues {

 export function joinByLabels(options: JoinByLabelsTransformOptions, data: DataFrame[]): DataFrame {
  if (!options.value?.length) {
-    return getErrorFrame('No value labele configured');
+    return getErrorFrame('No value label configured');
  }
  const distinctLabels = getDistinctLabels(data);
  if (distinctLabels.size < 1) {
@ -104,7 +104,11 @@ export function joinByLabels(options: JoinByLabelsTransformOptions, data: DataFr
    }
  }

-  const frame: DataFrame = { fields: [], length: nameValues[0].length };
+  const frame: DataFrame = {
+    fields: [],
+    length: nameValues[0].length,
+    refId: `${DataTransformerID.joinByLabels}-${data.map((frame) => frame.refId).join('-')}`,
+  };
  for (let i = 0; i < join.length; i++) {
    frame.fields.push({
      name: join[i],
--- a/public/app/features/transformers/rowsToFields/rowsToFields.test.ts
+++ b/public/app/features/transformers/rowsToFields/rowsToFields.test.ts
@ -12,6 +12,7 @@ describe('Rows to fields', () => {
        { name: 'Miiin', type: FieldType.number, values: [3, 100] },
        { name: 'max', type: FieldType.string, values: [15, 200] },
      ],
+      refId: 'A',
    });

    const result = rowsToFields(
@ -57,6 +58,7 @@ describe('Rows to fields', () => {
          },
        ],
        "length": 1,
+        "refId": "rowsToFields-A",
      }
    `);
  });
--- a/public/app/features/transformers/rowsToFields/rowsToFields.ts
+++ b/public/app/features/transformers/rowsToFields/rowsToFields.ts
@ -64,6 +64,7 @@ export function rowsToFields(options: RowToFieldsTransformOptions, data: DataFra
  return {
    fields: outFields,
    length: 1,
+    refId: `${DataTransformerID.rowsToFields}-${data.refId}`,
  };
 }

--- a/public/app/plugins/panel/geomap/editor/FrameSelectionEditor.tsx
+++ b/public/app/plugins/panel/geomap/editor/FrameSelectionEditor.tsx
@ -29,7 +29,9 @@ export const FrameSelectionEditor = ({ value, context, onChange }: Props) => {
  );
 };

-export const FrameMultiSelectionEditor = ({ value, context, onChange }: Props) => {
+type FrameMultiSelectionEditorProps = Omit<StandardEditorProps<MatcherConfig>, 'item'>;
+
+export const FrameMultiSelectionEditor = ({ value, context, onChange }: FrameMultiSelectionEditorProps) => {
  const onFilterChange = useCallback(
    (v: string[]) => {
      onChange(