Transformations: Selectively apply transformation to queries (#61735)

devenv/dev-dashboards/transforms/extract-json-paths.json

@@ -299,7 +299,10 @@
   "revision": 1,
   "schemaVersion": 37,
   "style": "dark",
-  "tags": [],
+  "tags": [
+    "gdev",
+    "transform"
+  ],
   "templating": {
     "list": []
   },
@@ -309,7 +312,7 @@
   },
   "timepicker": {},
   "timezone": "",
-  "title": "Test extractFields JSON",
+  "title": "Transforms - Test extractFields JSON",
   "uid": "pD4vPYhVz",
   "version": 3,
   "weekStart": ""

devenv/dev-dashboards/transforms/filter.json

@@ -0,0 +1,171 @@
+{
+    "annotations": {
+      "list": [
+        {
+          "builtIn": 1,
+          "datasource": {
+            "type": "grafana",
+            "uid": "-- Grafana --"
+          },
+          "enable": true,
+          "hide": true,
+          "iconColor": "rgba(0, 211, 255, 1)",
+          "name": "Annotations & Alerts",
+          "target": {
+            "limit": 100,
+            "matchAny": false,
+            "tags": [],
+            "type": "dashboard"
+          },
+          "type": "dashboard"
+        }
+      ]
+    },
+    "editable": true,
+    "fiscalYearStartMonth": 0,
+    "graphTooltip": 0,
+    "id": 1394,
+    "links": [],
+    "liveNow": false,
+    "panels": [
+      {
+        "datasource": {
+          "uid": "PD8C576611E62080A",
+          "type": "testdata"
+        },
+        "fieldConfig": {
+          "defaults": {
+            "custom": {
+              "align": "auto",
+              "cellOptions": {
+                "type": "auto"
+              },
+              "inspect": false
+            },
+            "mappings": [],
+            "thresholds": {
+              "mode": "absolute",
+              "steps": [
+                {
+                  "color": "green"
+                },
+                {
+                  "color": "red",
+                  "value": 80
+                }
+              ]
+            },
+            "color": {
+              "mode": "thresholds"
+            }
+          },
+          "overrides": []
+        },
+        "gridPos": {
+          "h": 9,
+          "w": 12,
+          "x": 0,
+          "y": 0
+        },
+        "id": 2,
+        "targets": [
+          {
+            "scenarioId": "csv_content",
+            "refId": "A",
+            "datasource": {
+              "uid": "PD8C576611E62080A",
+              "type": "testdata"
+            },
+            "csvContent": "AAA\n1\n2\n3\n4"
+          },
+          {
+            "scenarioId": "csv_content",
+            "refId": "B",
+            "datasource": {
+              "uid": "PD8C576611E62080A",
+              "type": "testdata"
+            },
+            "csvContent": "BBB\n1\n2\n3\n4\n",
+            "hide": false
+          }
+        ],
+        "title": "Transformer query filters",
+        "type": "table",
+        "transformations": [
+          {
+            "id": "reduce",
+            "options": {
+              "reducers": [
+                "min"
+              ],
+              "mode": "reduceFields",
+              "includeTimeField": false
+            },
+            "filter": {
+              "id": "byRefId",
+              "options": "A"
+            }
+          },
+          {
+            "id": "reduce",
+            "options": {
+              "reducers": [
+                "max"
+              ],
+              "mode": "reduceFields",
+              "includeTimeField": false
+            },
+            "filter": {
+              "id": "byRefId",
+              "options": "B"
+            }
+          },
+          {
+            "id": "concatenate",
+            "options": {}
+          },
+          {
+            "id": "organize",
+            "options": {
+              "excludeByName": {},
+              "indexByName": {},
+              "renameByName": {
+                "AAA": "Min from Query A",
+                "BBB": "Max from Query B"
+              }
+            }
+          }
+        ],
+        "options": {
+          "showHeader": true,
+          "footer": {
+            "show": false,
+            "reducer": [
+              "sum"
+            ],
+            "countRows": false,
+            "fields": ""
+          },
+          "frameIndex": 0
+        },
+        "pluginVersion": "9.4.0-pre"
+      }
+    ],
+    "schemaVersion": 37,
+    "style": "dark",
+    "tags": [
+      "gdev",
+      "transform"
+    ],
+    "templating": {
+      "list": []
+    },
+    "time": {
+      "from": "now-6h",
+      "to": "now"
+    },
+    "timepicker": {},
+    "timezone": "",
+    "title": "Transforms - Filters",
+    "uid": "fGWBVW4k"
+  }

devenv/dev-dashboards/transforms/join-by-field.json

@@ -624,7 +624,7 @@
   },
   "timepicker": {},
   "timezone": "",
-  "title": "Join by field",
+  "title": "Transforms - Join by field",
   "uid": "gw0K4rmVz",
   "version": 6,
   "weekStart": ""

devenv/dev-dashboards/transforms/join-by-labels.json

@@ -347,7 +347,7 @@
   },
   "timepicker": {},
   "timezone": "",
-  "title": "Join by labels",
+  "title": "Transforms - Join by labels",
   "uid": "FVl-9CR4z",
   "version": 10,
   "weekStart": ""

devenv/dev-dashboards/transforms/reuse.json

@@ -521,7 +521,10 @@
     ],
     "schemaVersion": 37,
     "style": "dark",
-    "tags": ["devenv"],
+    "tags": [
+      "gdev",
+      "transform"
+    ],
     "templating": {
       "list": []
     },
@@ -531,6 +534,6 @@
     },
     "timepicker": {},
     "timezone": "",
-    "title": "Reuse dashboard queries",
+    "title": "Transforms - Reuse dashboard queries",
     "uid": "fYGWTVW4k"
   }

devenv/jsonnet/dev-dashboards.libsonnet

@@ -184,6 +184,13 @@ local dashboard = grafana.dashboard;
         id: 0,
       }
     },
+    dashboard.new('filter', import '../dev-dashboards/transforms/filter.json') +
+    resource.addMetadata('folder', 'dev-dashboards') +
+    {
+      spec+: {
+        id: 0,
+      }
+    },
     dashboard.new('gauge-multi-series', import '../dev-dashboards/panel-gauge/gauge-multi-series.json') +
     resource.addMetadata('folder', 'dev-dashboards') +
     {

kinds/dashboard/dashboard_kind.cue

@@ -285,7 +285,10 @@ lineage: seqs: [
 				// TODO docs
 				// FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it
 				#Transformation: {
-					id: string
+					id:   string
+					hide: bool | *false
+					// only apply to some frames
+					filter?: #MatcherConfig
 					options: {...}
 				} @cuetsy(kind="interface") @grafanamaturity(NeedsExpertReview)
 

packages/grafana-data/src/transformations/matchers/ids.ts

@@ -35,7 +35,6 @@ export enum FrameMatcherID {
   byName = 'byName',
   byRefId = 'byRefId',
   byIndex = 'byIndex',
-  byLabel = 'byLabel',
 }
 
 /**

packages/grafana-data/src/transformations/transformDataFrame.test.ts

@@ -3,10 +3,11 @@ import { FieldType } from '../types';
 import { mockTransformationsRegistry } from '../utils/tests/mockTransformationsRegistry';
 
 import { ReducerID } from './fieldReducer';
+import { FrameMatcherID } from './matchers/ids';
 import { transformDataFrame } from './transformDataFrame';
 import { filterFieldsByNameTransformer } from './transformers/filterByName';
 import { DataTransformerID } from './transformers/ids';
-import { reduceTransformer } from './transformers/reduce';
+import { reduceTransformer, ReduceTransformerMode } from './transformers/reduce';
 
 const seriesAWithSingleField = toDataFrame({
   name: 'A',
@@ -73,4 +74,44 @@ describe('transformDataFrame', () => {
       expect(processed[0].fields[0].values.get(0)).toEqual('temperature');
     });
   });
+
+  it('Support filtering', async () => {
+    const frameA = toDataFrame({
+      refId: 'A',
+      fields: [{ name: 'value', type: FieldType.number, values: [5, 6] }],
+    });
+    const frameB = toDataFrame({
+      refId: 'B',
+      fields: [{ name: 'value', type: FieldType.number, values: [7, 8] }],
+    });
+
+    const cfg = [
+      {
+        id: DataTransformerID.reduce,
+        filter: {
+          id: FrameMatcherID.byRefId,
+          options: 'A', // Only apply to A
+        },
+        options: {
+          reducers: [ReducerID.first],
+          mode: ReduceTransformerMode.ReduceFields,
+        },
+      },
+    ];
+
+    // Only apply A
+    await expect(transformDataFrame(cfg, [frameA, frameB])).toEmitValuesWith((received) => {
+      const processed = received[0].map((v) => v.fields[0].values.toArray());
+      expect(processed).toBeTruthy();
+      expect(processed).toMatchObject([[5], [7, 8]]);
+    });
+
+    // Only apply to B
+    cfg[0].filter.options = 'B';
+    await expect(transformDataFrame(cfg, [frameA, frameB])).toEmitValuesWith((received) => {
+      const processed = received[0].map((v) => v.fields[0].values.toArray());
+      expect(processed).toBeTruthy();
+      expect(processed).toMatchObject([[5, 6], [7]]);
+    });
+  });
 });

packages/grafana-data/src/transformations/transformDataFrame.ts

@@ -1,8 +1,9 @@
 import { MonoTypeOperatorFunction, Observable, of } from 'rxjs';
 import { map, mergeMap } from 'rxjs/operators';
 
-import { DataFrame, DataTransformContext, DataTransformerConfig } from '../types';
+import { DataFrame, DataTransformContext, DataTransformerConfig, FrameMatcher } from '../types';
 
+import { getFrameMatchers } from './matchers';
 import { standardTransformersRegistry, TransformerRegistryItem } from './standardTransformersRegistry';
 
 const getOperator =
@@ -17,15 +18,30 @@ const getOperator =
     const defaultOptions = info.transformation.defaultOptions ?? {};
     const options = { ...defaultOptions, ...config.options };
 
+    const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
     return source.pipe(
       mergeMap((before) =>
-        of(before).pipe(info.transformation.operator(options, ctx), postProcessTransform(before, info))
+        of(filterInput(before, matcher)).pipe(
+          info.transformation.operator(options, ctx),
+          postProcessTransform(before, info, matcher)
+        )
       )
     );
   };
 
+function filterInput(data: DataFrame[], matcher?: FrameMatcher) {
+  if (matcher) {
+    return data.filter((v) => matcher(v));
+  }
+  return data;
+}
+
 const postProcessTransform =
-  (before: DataFrame[], info: TransformerRegistryItem<any>): MonoTypeOperatorFunction<DataFrame[]> =>
+  (
+    before: DataFrame[],
+    info: TransformerRegistryItem<any>,
+    matcher?: FrameMatcher
+  ): MonoTypeOperatorFunction<DataFrame[]> =>
   (source) =>
     source.pipe(
       map((after) => {
@@ -46,6 +62,21 @@ const postProcessTransform =
           }
         }
 
+        // Add back the filtered out frames
+        if (matcher) {
+          // keep the frame order the same
+          let insert = 0;
+          const append = before.filter((v, idx) => {
+            const keep = !matcher(v);
+            if (keep && !insert) {
+              insert = idx;
+            }
+            return keep;
+          });
+          if (append.length) {
+            after.splice(insert, 0, ...append);
+          }
+        }
         return after;
       })
     );

packages/grafana-data/src/types/transformations.ts

@@ -1,11 +1,15 @@
-export type { MatcherConfig } from '@grafana/schema';
 import { MonoTypeOperatorFunction } from 'rxjs';
 
+import { MatcherConfig } from '@grafana/schema';
+
 import { RegistryItemWithOptions } from '../utils/Registry';
 
 import { DataFrame, Field } from './dataFrame';
 import { InterpolateFunction } from './panel';
 
+/** deprecated, use it from schema */
+export type { MatcherConfig };
+
 /**
  * Context passed to transformDataFrame and to each transform operator
  */
@@ -44,10 +48,15 @@ export interface DataTransformerConfig<TOptions = any> {
    * Unique identifier of transformer
    */
   id: string;
+
   /**
    * Disabled transformations are skipped
    */
   disabled?: boolean;
+
+  /** Optional frame matcher.  When missing it will be applied to all results  */
+  filter?: MatcherConfig;
+
   /**
    * Options to be passed to the transformer
    */

packages/grafana-schema/src/index.gen.ts

@@ -44,6 +44,7 @@ export {
   defaultThresholdsConfig,
   MappingType,
   SpecialValueMatch,
+  defaultTransformation,
   DashboardCursorSync,
   defaultDashboardCursorSync,
   defaultRowPanel

packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts

@@ -356,10 +356,19 @@ export interface ValueMappingResult {
  * FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it
  */
 export interface Transformation {
+  /**
+   * only apply to some frames
+   */
+  filter?: MatcherConfig;
+  hide: boolean;
   id: string;
   options: Record<string, unknown>;
 }
 
+export const defaultTransformation: Partial<Transformation> = {
+  hide: false,
+};
+
 /**
  * 0 for no shared crosshair or tooltip (default).
  * 1 for shared crosshair.

pkg/kinds/dashboard/dashboard_types_gen.go

@@ -710,6 +710,8 @@ type ThresholdsMode string
 // TODO docs
 // FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it
 type Transformation struct {
+	Filter  *MatcherConfig         `json:"filter,omitempty"`
+	Hide    bool                   `json:"hide"`
 	Id      string                 `json:"id"`
 	Options map[string]interface{} `json:"options"`
 }

public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx

@@ -8,6 +8,7 @@ import {
   GrafanaTheme2,
   transformDataFrame,
   TransformerRegistryItem,
+  getFrameMatchers,
 } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
 import { Icon, JSONFormatter, useStyles2 } from '@grafana/ui';
@@ -37,9 +38,16 @@ export const TransformationEditor = ({
   const config = useMemo(() => configs[index], [configs, index]);
 
   useEffect(() => {
+    const config = configs[index].transformation;
+    const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
     const inputTransforms = configs.slice(0, index).map((t) => t.transformation);
     const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation);
-    const inputSubscription = transformDataFrame(inputTransforms, data).subscribe(setInput);
+    const inputSubscription = transformDataFrame(inputTransforms, data).subscribe((v) => {
+      if (matcher) {
+        v = data.filter((v) => matcher(v));
+      }
+      setInput(v);
+    });
     const outputSubscription = transformDataFrame(inputTransforms, data)
       .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before)))
       .subscribe(setOutput);
@@ -56,18 +64,13 @@ export const TransformationEditor = ({
         options: { ...uiConfig.transformation.defaultOptions, ...config.transformation.options },
         input,
         onChange: (opts) => {
-          onChange(index, { id: config.transformation.id, options: opts });
+          onChange(index, {
+            ...config.transformation,
+            options: opts,
+          });
         },
       }),
-    [
+    [uiConfig.editor, uiConfig.transformation.defaultOptions, config.transformation, input, onChange, index]
-      uiConfig.editor,
-      uiConfig.transformation.defaultOptions,
-      config.transformation.options,
-      config.transformation.id,
-      input,
-      onChange,
-      index,
-    ]
   );
 
   return (

public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx

@@ -0,0 +1,55 @@
+import { css } from '@emotion/css';
+import React, { useMemo } from 'react';
+
+import {
+  DataFrame,
+  DataTransformerConfig,
+  GrafanaTheme2,
+  StandardEditorContext,
+  StandardEditorsRegistryItem,
+} from '@grafana/data';
+import { useStyles2 } from '@grafana/ui';
+import { FrameSelectionEditor } from 'app/plugins/panel/geomap/editor/FrameSelectionEditor';
+
+interface TransformationFilterProps {
+  index: number;
+  config: DataTransformerConfig;
+  data: DataFrame[];
+  onChange: (index: number, config: DataTransformerConfig) => void;
+}
+
+export const TransformationFilter = ({ index, data, config, onChange }: TransformationFilterProps) => {
+  const styles = useStyles2(getStyles);
+  const context = useMemo(() => {
+    // eslint-disable-next-line
+    return { data } as StandardEditorContext<unknown>;
+  }, [data]);
+
+  return (
+    <div className={styles.wrapper}>
+      <h5>Apply tranformation to</h5>
+      <FrameSelectionEditor
+        value={config.filter!}
+        context={context}
+        // eslint-disable-next-line
+        item={{} as StandardEditorsRegistryItem}
+        onChange={(filter) => onChange(index, { ...config, filter })}
+      />
+    </div>
+  );
+};
+
+const getStyles = (theme: GrafanaTheme2) => {
+  const borderRadius = theme.shape.borderRadius();
+
+  return {
+    wrapper: css`
+      padding: ${theme.spacing(2)};
+      border: 2px solid ${theme.colors.background.secondary};
+      border-top: none;
+      border-radius: 0 0 ${borderRadius} ${borderRadius};
+      position: relative;
+      top: -4px;
+    `,
+  };
+};

public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx

@@ -1,7 +1,7 @@
 import React, { useCallback } from 'react';
 import { useToggle } from 'react-use';
 
-import { DataFrame, DataTransformerConfig, TransformerRegistryItem } from '@grafana/data';
+import { DataFrame, DataTransformerConfig, TransformerRegistryItem, FrameMatcherID } from '@grafana/data';
 import { HorizontalGroup } from '@grafana/ui';
 import { OperationRowHelp } from 'app/core/components/QueryOperationRow/OperationRowHelp';
 import { QueryOperationAction } from 'app/core/components/QueryOperationRow/QueryOperationAction';
@@ -12,6 +12,7 @@ import {
 import { PluginStateInfo } from 'app/features/plugins/components/PluginStateInfo';
 
 import { TransformationEditor } from './TransformationEditor';
+import { TransformationFilter } from './TransformationFilter';
 import { TransformationsEditorTransformation } from './types';
 
 interface TransformationOperationRowProps {
@@ -24,7 +25,7 @@ interface TransformationOperationRowProps {
   onChange: (index: number, config: DataTransformerConfig) => void;
 }
 
-export const TransformationOperationRow: React.FC<TransformationOperationRowProps> = ({
+export const TransformationOperationRow = ({
   onRemove,
   index,
   id,
@@ -32,10 +33,12 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
   configs,
   uiConfig,
   onChange,
-}) => {
+}: TransformationOperationRowProps) => {
   const [showDebug, toggleDebug] = useToggle(false);
   const [showHelp, toggleHelp] = useToggle(false);
   const disabled = configs[index].transformation.disabled;
+  const filter = configs[index].transformation.filter != null;
+  const showFilter = filter || data.length > 1;
 
   const onDisableToggle = useCallback(
     (index: number) => {
@@ -48,6 +51,20 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
     [onChange, configs]
   );
 
+  // Adds or removes the frame filter
+  const toggleFilter = useCallback(() => {
+    let current = { ...configs[index].transformation };
+    if (current.filter) {
+      delete current.filter;
+    } else {
+      current.filter = {
+        id: FrameMatcherID.byRefId,
+        options: '', // empty string will not do anything
+      };
+    }
+    onChange(index, current);
+  }, [onChange, index, configs]);
+
   const renderActions = ({ isOpen }: QueryOperationRowRenderProps) => {
     return (
       <HorizontalGroup align="center" width="auto">
@@ -58,6 +75,7 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
           onClick={toggleHelp}
           active={showHelp}
         />
+        {showFilter && <QueryOperationAction title="Filter" icon="filter" onClick={toggleFilter} active={filter} />}
         <QueryOperationAction title="Debug" disabled={!isOpen} icon="bug" onClick={toggleDebug} active={showDebug} />
         <QueryOperationAction
           title="Disable/Enable transformation"
@@ -80,6 +98,9 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
       disabled={disabled}
     >
       {showHelp && <OperationRowHelp markdown={prepMarkdown(uiConfig)} />}
+      {filter && (
+        <TransformationFilter index={index} config={configs[index].transformation} data={data} onChange={onChange} />
+      )}
       <TransformationEditor
         debugMode={showDebug}
         index={index}

public/app/features/transformers/editors/FilterByRefIdTransformerEditor.tsx

@@ -8,7 +8,7 @@ import {
   TransformerUIProps,
 } from '@grafana/data';
 import { FilterFramesByRefIdTransformerOptions } from '@grafana/data/src/transformations/transformers/filterByRefId';
-import { HorizontalGroup, FilterPill } from '@grafana/ui';
+import { HorizontalGroup, FilterPill, FieldValidationMessage } from '@grafana/ui';
 
 interface FilterByRefIdTransformerEditorProps extends TransformerUIProps<FilterFramesByRefIdTransformerOptions> {}
 
@@ -103,28 +103,36 @@ export class FilterByRefIdTransformerEditor extends React.PureComponent<
 
   render() {
     const { options, selected } = this.state;
+    const { input } = this.props;
     return (
-      <div className="gf-form-inline">
+      <>
-        <div className="gf-form gf-form--grow">
+        {input.length <= 1 && (
-          <div className="gf-form-label width-8">Series refId</div>
+          <div>
-          <HorizontalGroup spacing="xs" align="flex-start" wrap>
+            <FieldValidationMessage>Filter data by query expects multiple queries in the input.</FieldValidationMessage>
-            {options.map((o, i) => {
+          </div>
-              const label = `${o.refId}${o.count > 1 ? ' (' + o.count + ')' : ''}`;
+        )}
-              const isSelected = selected.indexOf(o.refId) > -1;
+        <div className="gf-form-inline">
-              return (
+          <div className="gf-form gf-form--grow">
-                <FilterPill
+            <div className="gf-form-label width-8">Series refId</div>
-                  key={`${o.refId}/${i}`}
+            <HorizontalGroup spacing="xs" align="flex-start" wrap>
-                  onClick={() => {
+              {options.map((o, i) => {
-                    this.onFieldToggle(o.refId);
+                const label = `${o.refId}${o.count > 1 ? ' (' + o.count + ')' : ''}`;
-                  }}
+                const isSelected = selected.indexOf(o.refId) > -1;
-                  label={label}
+                return (
-                  selected={isSelected}
+                  <FilterPill
-                />
+                    key={`${o.refId}/${i}`}
-              );
+                    onClick={() => {
-            })}
+                      this.onFieldToggle(o.refId);
-          </HorizontalGroup>
+                    }}
+                    label={label}
+                    selected={isSelected}
+                  />
+                );
+              })}
+            </HorizontalGroup>
+          </div>
         </div>
-      </div>
+      </>
     );
   }
 }

public/app/features/transformers/editors/OrganizeFieldsTransformerEditor.tsx

@@ -17,8 +17,7 @@ import { useAllFieldNamesFromDataFrames } from '../utils';
 
 interface OrganizeFieldsTransformerEditorProps extends TransformerUIProps<OrganizeFieldsTransformerOptions> {}
 
-const OrganizeFieldsTransformerEditor: React.FC<OrganizeFieldsTransformerEditorProps> = (props) => {
+const OrganizeFieldsTransformerEditor = ({ options, input, onChange }: OrganizeFieldsTransformerEditorProps) => {
-  const { options, input, onChange } = props;
   const { indexByName, excludeByName, renameByName } = options;
 
   const fieldNames = useAllFieldNamesFromDataFrames(input);
@@ -75,7 +74,8 @@ const OrganizeFieldsTransformerEditor: React.FC<OrganizeFieldsTransformerEditorP
   if (input.length > 1) {
     return (
       <FieldValidationMessage>
-        Organize fields only works with a single frame. Consider applying a join transformation first.
+        Organize fields only works with a single frame. Consider applying a join transformation or filtering the input
+        first.
       </FieldValidationMessage>
     );
   }