Transformations: Extended support for variables in filter by name (#75734)

* Extend support for variables in filter by name * Simlpify help and include variable support * Simplify regexp * Remove id that was left from an erlier implementation attempt * Update docs/sources/panels-visualizations/query-transform-data/transform-data/index.md Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com> * Improve variable name and fix react warning --------- Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com>
2025-02-25 18:55:37 -06:00 · 2023-10-03 15:06:08 +02:00 · 2023-10-03 15:06:08 +02:00 · e0919a340e
commit e0919a340e
parent 1456b26075
7 changed files with 184 additions and 37 deletions
--- a/.betterer.results
+++ b/.betterer.results
@ -5265,9 +5265,6 @@ exports[`better eslint`] = {
    "public/app/features/transformers/editors/ConvertFieldTypeTransformerEditor.tsx:5381": [
      [0, 0, 0, "Do not use any type assertions.", "0"]
    ],
-    "public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx:5381": [
-      [0, 0, 0, "Styles should be written using objects.", "0"]
-    ],
    "public/app/features/transformers/editors/GroupByTransformerEditor.tsx:5381": [
      [0, 0, 0, "Do not use any type assertions.", "0"],
      [0, 0, 0, "Styles should be written using objects.", "1"],
--- a/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md
+++ b/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md
@ -281,28 +281,41 @@ You'll get the following output:

 ### Filter by name

-Use this transformation to remove portions of the query results.
+Use this transformation to remove parts of the query results.

-Grafana displays the **Identifier** field, followed by the fields returned by your query.
+You can filter field names in three different ways:

-You can apply filters in one of two ways:
+- [Using a regular expression](#use-a-regular-expression)
+- [Manually selecting included fields](#manually-select-included-fields)
+- [Using a dashboard variable](#use-a-dashboard-variable)

- Enter a regex expression.
- Click a field to toggle filtering on that field. Filtered fields are displayed with dark gray text, unfiltered fields have white text.
+#### Use a regular expression

-In the example below, I removed the Min field from the results.
+When you filter using a regular expression, field names that match the regular expression are included.

-Here is the original query table. (This is streaming data, so numbers change over time and between screenshots.)
+From the input data:

-{{< figure src="/static/img/docs/transformations/filter-name-table-before-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+| Time                | dev-eu-west | dev-eu-north | prod-eu-west | prod-eu-north |
+| ------------------- | ----------- | ------------ | ------------ | ------------- |
+| 2023-03-04 23:56:23 | 23.5        | 24.5         | 22.2         | 20.2          |
+| 2023-03-04 23:56:23 | 23.6        | 24.4         | 22.1         | 20.1          |

-Here is the table after I applied the transformation to remove the Min field.
+The result from using the regular expression `prod.*` would be:

-{{< figure src="/static/img/docs/transformations/filter-name-table-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+| Time                | prod-eu-west | prod-eu-north |
+| ------------------- | ------------ | ------------- |
+| 2023-03-04 23:56:23 | 22.2         | 20.2          |
+| 2023-03-04 23:56:23 | 22.1         | 20.1          |

-Here is the same query using a Stat visualization.
+The regular expression can include an interpolated dashboard variable by using the `${[variable name]}` syntax.

-{{< figure src="/static/img/docs/transformations/filter-name-stat-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}}
+#### Manually select included fields
+
+Click and uncheck the field names to remove them from the result. Fields that are matched by the regular expression are still included, even if they're unchecked.
+
+#### Use a dashboard variable
+
+Enable `From variable` to let you select a dashboard variable that's used to include fields. By setting up a [dashboard variable][] with multiple choices, the same fields can be displayed across multiple visualizations.

 ### Filter data by query

@ -1010,4 +1023,8 @@ Use this transformation to format the output of a time field. Output can be form

 [feature toggle]: "/docs/grafana/ -> /docs/grafana/<GRAFANA VERSION>/setup-grafana/configure-grafana#feature_toggles"
 [feature toggle]: "/docs/grafana-cloud/ -> /docs/grafana/<GRAFANA VERSION>/setup-grafana/configure-grafana#feature_toggles"
+
+[dashboard variable]: "/docs/grafana/ -> docs/grafana/<GRAFANA VERSION>/dashboards/variables"
+[dashboard variable]: "/docs/grafana-cloud/ -> docs/grafana/<GRAFANA VERSION>/dashboards/variables"
+
 {{% /docs/reference %}}
--- a/packages/grafana-data/src/transformations/matchers/nameMatcher.ts
+++ b/packages/grafana-data/src/transformations/matchers/nameMatcher.ts
@ -8,6 +8,7 @@ import { FieldMatcherID, FrameMatcherID } from './ids';
 export interface RegexpOrNamesMatcherOptions {
  pattern?: string;
  names?: string[];
+  variable?: string;
 }

 /**
--- a/packages/grafana-data/src/transformations/transformers/filterByName.test.ts
+++ b/packages/grafana-data/src/transformations/transformers/filterByName.test.ts
@ -196,8 +196,83 @@ describe('filterByName transformer', () => {
        expect(filtered.fields[0].name).toBe('B');
      });
    });
+    it('it can use a variable with multiple comma separated', async () => {
+      const cfg = {
+        id: DataTransformerID.filterFieldsByName,
+        options: {
+          include: {
+            variable: '$var',
+          },
+          byVariable: true,
+        },
+      };

-    it('uses template variable substituion', async () => {
+      const ctx = {
+        interpolate: (target: string | undefined, scopedVars?: ScopedVars, format?: string | Function): string => {
+          if (!target) {
+            return '';
+          }
+          const variables: ScopedVars = {
+            var: {
+              value: 'B,D',
+              text: 'Test',
+            },
+          };
+          for (const key of Object.keys(variables)) {
+            return target.replace(`$${key}`, variables[key]!.value);
+          }
+          return target;
+        },
+      };
+
+      await expect(transformDataFrame([cfg], [seriesWithNamesToMatch], ctx)).toEmitValuesWith((received) => {
+        const data = received[0];
+        const filtered = data[0];
+        expect(filtered.fields.length).toBe(2);
+        expect(filtered.fields[0].name).toBe('B');
+        expect(filtered.fields[1].name).toBe('D');
+      });
+    });
+
+    it('it can use a variable with multiple comma separated values in {}', async () => {
+      const cfg = {
+        id: DataTransformerID.filterFieldsByName,
+        options: {
+          include: {
+            variable: '$var',
+          },
+          byVariable: true,
+        },
+      };
+
+      const ctx = {
+        interpolate: (target: string | undefined, scopedVars?: ScopedVars, format?: string | Function): string => {
+          if (!target) {
+            return '';
+          }
+          const variables: ScopedVars = {
+            var: {
+              value: '{B,D}',
+              text: 'Test',
+            },
+          };
+          for (const key of Object.keys(variables)) {
+            return target.replace(`$${key}`, variables[key]!.value);
+          }
+          return target;
+        },
+      };
+
+      await expect(transformDataFrame([cfg], [seriesWithNamesToMatch], ctx)).toEmitValuesWith((received) => {
+        const data = received[0];
+        const filtered = data[0];
+        expect(filtered.fields.length).toBe(2);
+        expect(filtered.fields[0].name).toBe('B');
+        expect(filtered.fields[1].name).toBe('D');
+      });
+    });
+
+    it('uses template variable substitution', async () => {
      const cfg = {
        id: DataTransformerID.filterFieldsByName,
        options: {
--- a/packages/grafana-data/src/transformations/transformers/filterByName.ts
+++ b/packages/grafana-data/src/transformations/transformers/filterByName.ts
@ -1,4 +1,4 @@
-import { DataTransformerInfo, MatcherConfig } from '../../types/transformations';
+import { DataTransformContext, DataTransformerInfo, MatcherConfig } from '../../types/transformations';
 import { FieldMatcherID } from '../matchers/ids';
 import { RegexpOrNamesMatcherOptions } from '../matchers/nameMatcher';

@ -8,6 +8,7 @@ import { DataTransformerID } from './ids';
 export interface FilterFieldsByNameTransformerOptions {
  include?: RegexpOrNamesMatcherOptions;
  exclude?: RegexpOrNamesMatcherOptions;
+  byVariable?: boolean;
 }

 export const filterFieldsByNameTransformer: DataTransformerInfo<FilterFieldsByNameTransformerOptions> = {
@ -20,25 +21,38 @@ export const filterFieldsByNameTransformer: DataTransformerInfo<FilterFieldsByNa
   * Return a modified copy of the series. If the transform is not or should not
   * be applied, just return the input series
   */
-  operator: (options, replace) => (source) =>
+  operator: (options, ctx) => (source) =>
    source.pipe(
      filterFieldsTransformer.operator(
        {
-          include: getMatcherConfig(options.include),
-          exclude: getMatcherConfig(options.exclude),
+          include: getMatcherConfig(ctx, options.include, options.byVariable),
+          exclude: getMatcherConfig(ctx, options.exclude, options.byVariable),
        },
-        replace
+        ctx
      )
    ),
 };

 // Exported to share with other implementations, but not exported to `@grafana/data`
-export const getMatcherConfig = (options?: RegexpOrNamesMatcherOptions): MatcherConfig | undefined => {
+export const getMatcherConfig = (
+  ctx: DataTransformContext,
+  options?: RegexpOrNamesMatcherOptions,
+  byVariable?: boolean
+): MatcherConfig | undefined => {
  if (!options) {
    return undefined;
  }

-  const { names, pattern } = options;
+  const { names, pattern, variable } = options;
+
+  if (byVariable && variable) {
+    const stringOfNames = ctx.interpolate(variable);
+    if (/\{.*\}/.test(stringOfNames)) {
+      const namesFromString = stringOfNames.slice(1).slice(0, -1).split(',');
+      return { id: FieldMatcherID.byNames, options: { names: namesFromString } };
+    }
+    return { id: FieldMatcherID.byNames, options: { names: stringOfNames.split(',') } };
+  }

  if ((!Array.isArray(names) || names.length === 0) && !pattern) {
    return undefined;
--- a/public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx
+++ b/public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx
@ -1,4 +1,3 @@
-import { css } from '@emotion/css';
 import React from 'react';

 import {
@ -10,9 +9,11 @@ import {
  getFieldDisplayName,
  stringToJsRegex,
  TransformerCategory,
+  SelectableValue,
 } from '@grafana/data';
 import { FilterFieldsByNameTransformerOptions } from '@grafana/data/src/transformations/transformers/filterByName';
-import { Field, Input, FilterPill, HorizontalGroup } from '@grafana/ui';
+import { getTemplateSrv } from '@grafana/runtime/src/services';
+import { Input, FilterPill, InlineFieldRow, InlineField, InlineSwitch, Select } from '@grafana/ui';

 interface FilterByNameTransformerEditorProps extends TransformerUIProps<FilterFieldsByNameTransformerOptions> {}

@ -21,6 +22,9 @@ interface FilterByNameTransformerEditorState {
  options: FieldNameInfo[];
  selected: string[];
  regex?: string;
+  variable?: string;
+  variables: SelectableValue[];
+  byVariable: boolean;
  isRegexValid?: boolean;
 }

@ -37,7 +41,10 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
    this.state = {
      include: props.options.include?.names || [],
      regex: props.options.include?.pattern,
+      variable: props.options.include?.variable,
+      byVariable: props.options.byVariable || false,
      options: [],
+      variables: [],
      selected: [],
      isRegexValid: true,
    };
@ -57,6 +64,9 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
    const { input, options } = this.props;
    const configuredOptions = Array.from(options.include?.names ?? []);

+    const variables = getTemplateSrv()
+      .getVariables()
+      .map((v) => ({ label: '$' + v.name, value: '$' + v.name }));
    const allNames: FieldNameInfo[] = [];
    const byName: KeyValue<FieldNameInfo> = {};

@ -97,12 +107,18 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
      this.setState({
        options: allNames,
        selected: selected.map((s) => s.name),
+        variables: variables,
+        byVariable: options.byVariable || false,
+        variable: options.include?.variable,
        regex: options.include?.pattern,
      });
    } else {
      this.setState({
        options: allNames,
        selected: allNames.map((n) => n.name),
+        variables: variables,
+        byVariable: options.byVariable || false,
+        variable: options.include?.variable,
        regex: options.include?.pattern,
      });
    }
@ -161,19 +177,46 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
    this.setState({ isRegexValid });
  };

+  onVariableChange = (selected: SelectableValue) => {
+    this.props.onChange({
+      ...this.props.options,
+      include: { variable: selected.value },
+    });
+
+    this.setState({ variable: selected.value });
+  };
+
+  onFromVariableChange = (e: React.FormEvent<HTMLInputElement>) => {
+    const val = e.currentTarget.checked;
+    this.props.onChange({ ...this.props.options, byVariable: val });
+    this.setState({ byVariable: val });
+  };
+
  render() {
    const { options, selected, isRegexValid } = this.state;
    return (
-      <div className="gf-form-inline">
-        <div className="gf-form gf-form--grow">
-          <div className="gf-form-label width-8">Identifier</div>
-          <HorizontalGroup spacing="xs" align="flex-start" wrap>
-            <Field
+      <div>
+        <InlineFieldRow label="Use variable">
+          <InlineField label="From variable">
+            <InlineSwitch value={this.state.byVariable} onChange={this.onFromVariableChange}></InlineSwitch>
+          </InlineField>
+        </InlineFieldRow>
+        {this.state.byVariable ? (
+          <InlineFieldRow>
+            <InlineField label="Variable">
+              <Select
+                value={this.state.variable}
+                onChange={this.onVariableChange}
+                options={this.state.variables || []}
+              ></Select>
+            </InlineField>
+          </InlineFieldRow>
+        ) : (
+          <InlineFieldRow label="Identifier">
+            <InlineField
+              label="Identifier"
              invalid={!isRegexValid}
              error={!isRegexValid ? 'Invalid pattern' : undefined}
-              className={css`
-                margin-bottom: 0;
-              `}
            >
              <Input
                placeholder="Regular expression pattern"
@ -182,7 +225,7 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
                onBlur={this.onInputBlur}
                width={25}
              />
-            </Field>
+            </InlineField>
            {options.map((o, i) => {
              const label = `${o.name}${o.count > 1 ? ' (' + o.count + ')' : ''}`;
              const isSelected = selected.indexOf(o.name) > -1;
@ -197,8 +240,8 @@ export class FilterByNameTransformerEditor extends React.PureComponent<
                />
              );
            })}
-          </HorizontalGroup>
-        </div>
+          </InlineFieldRow>
+        )}
      </div>
    );
  }
--- a/public/app/features/transformers/partitionByValues/partitionByValues.ts
+++ b/public/app/features/transformers/partitionByValues/partitionByValues.ts
@ -73,7 +73,7 @@ export const partitionByValuesTransformer: SynchronousDataTransformerInfo<Partit
    source.pipe(map((data) => partitionByValuesTransformer.transformer(options, ctx)(data))),

  transformer: (options: PartitionByValuesTransformerOptions, ctx: DataTransformContext) => {
-    const matcherConfig = getMatcherConfig({ names: options.fields });
+    const matcherConfig = getMatcherConfig(ctx, { names: options.fields });

    if (!matcherConfig) {
      return noopTransformer.transformer({}, ctx);