From e0919a340e2f5d57046ff6ff00b0ea1bc71f25e1 Mon Sep 17 00:00:00 2001 From: Oscar Kilhed Date: Tue, 3 Oct 2023 15:06:08 +0200 Subject: [PATCH] Transformations: Extended support for variables in filter by name (#75734) * Extend support for variables in filter by name * Simlpify help and include variable support * Simplify regexp * Remove id that was left from an erlier implementation attempt * Update docs/sources/panels-visualizations/query-transform-data/transform-data/index.md Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com> * Improve variable name and fix react warning --------- Co-authored-by: Isabel <76437239+imatwawana@users.noreply.github.com> --- .betterer.results | 3 - .../transform-data/index.md | 41 +++++++--- .../transformations/matchers/nameMatcher.ts | 1 + .../transformers/filterByName.test.ts | 77 ++++++++++++++++++- .../transformers/filterByName.ts | 28 +++++-- .../editors/FilterByNameTransformerEditor.tsx | 69 +++++++++++++---- .../partitionByValues/partitionByValues.ts | 2 +- 7 files changed, 184 insertions(+), 37 deletions(-) diff --git a/.betterer.results b/.betterer.results index 251f4c90001..8b826767eed 100644 --- a/.betterer.results +++ b/.betterer.results @@ -5265,9 +5265,6 @@ exports[`better eslint`] = { "public/app/features/transformers/editors/ConvertFieldTypeTransformerEditor.tsx:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"] ], - "public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx:5381": [ - [0, 0, 0, "Styles should be written using objects.", "0"] - ], "public/app/features/transformers/editors/GroupByTransformerEditor.tsx:5381": [ [0, 0, 0, "Do not use any type assertions.", "0"], [0, 0, 0, "Styles should be written using objects.", "1"], diff --git a/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md b/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md index 210b23b989d..5a5ea2a1931 100644 --- a/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md +++ b/docs/sources/panels-visualizations/query-transform-data/transform-data/index.md @@ -281,28 +281,41 @@ You'll get the following output: ### Filter by name -Use this transformation to remove portions of the query results. +Use this transformation to remove parts of the query results. -Grafana displays the **Identifier** field, followed by the fields returned by your query. +You can filter field names in three different ways: -You can apply filters in one of two ways: +- [Using a regular expression](#use-a-regular-expression) +- [Manually selecting included fields](#manually-select-included-fields) +- [Using a dashboard variable](#use-a-dashboard-variable) -- Enter a regex expression. -- Click a field to toggle filtering on that field. Filtered fields are displayed with dark gray text, unfiltered fields have white text. +#### Use a regular expression -In the example below, I removed the Min field from the results. +When you filter using a regular expression, field names that match the regular expression are included. -Here is the original query table. (This is streaming data, so numbers change over time and between screenshots.) +From the input data: -{{< figure src="/static/img/docs/transformations/filter-name-table-before-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} +| Time | dev-eu-west | dev-eu-north | prod-eu-west | prod-eu-north | +| ------------------- | ----------- | ------------ | ------------ | ------------- | +| 2023-03-04 23:56:23 | 23.5 | 24.5 | 22.2 | 20.2 | +| 2023-03-04 23:56:23 | 23.6 | 24.4 | 22.1 | 20.1 | -Here is the table after I applied the transformation to remove the Min field. +The result from using the regular expression `prod.*` would be: -{{< figure src="/static/img/docs/transformations/filter-name-table-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} +| Time | prod-eu-west | prod-eu-north | +| ------------------- | ------------ | ------------- | +| 2023-03-04 23:56:23 | 22.2 | 20.2 | +| 2023-03-04 23:56:23 | 22.1 | 20.1 | -Here is the same query using a Stat visualization. +The regular expression can include an interpolated dashboard variable by using the `${[variable name]}` syntax. -{{< figure src="/static/img/docs/transformations/filter-name-stat-after-7-0.png" class="docs-image--no-shadow" max-width= "1100px" >}} +#### Manually select included fields + +Click and uncheck the field names to remove them from the result. Fields that are matched by the regular expression are still included, even if they're unchecked. + +#### Use a dashboard variable + +Enable `From variable` to let you select a dashboard variable that's used to include fields. By setting up a [dashboard variable][] with multiple choices, the same fields can be displayed across multiple visualizations. ### Filter data by query @@ -1010,4 +1023,8 @@ Use this transformation to format the output of a time field. Output can be form [feature toggle]: "/docs/grafana/ -> /docs/grafana//setup-grafana/configure-grafana#feature_toggles" [feature toggle]: "/docs/grafana-cloud/ -> /docs/grafana//setup-grafana/configure-grafana#feature_toggles" + +[dashboard variable]: "/docs/grafana/ -> docs/grafana//dashboards/variables" +[dashboard variable]: "/docs/grafana-cloud/ -> docs/grafana//dashboards/variables" + {{% /docs/reference %}} diff --git a/packages/grafana-data/src/transformations/matchers/nameMatcher.ts b/packages/grafana-data/src/transformations/matchers/nameMatcher.ts index 4c4b3ba12b6..7befe20ee88 100644 --- a/packages/grafana-data/src/transformations/matchers/nameMatcher.ts +++ b/packages/grafana-data/src/transformations/matchers/nameMatcher.ts @@ -8,6 +8,7 @@ import { FieldMatcherID, FrameMatcherID } from './ids'; export interface RegexpOrNamesMatcherOptions { pattern?: string; names?: string[]; + variable?: string; } /** diff --git a/packages/grafana-data/src/transformations/transformers/filterByName.test.ts b/packages/grafana-data/src/transformations/transformers/filterByName.test.ts index 2b59037b61d..d6b4f7e839e 100644 --- a/packages/grafana-data/src/transformations/transformers/filterByName.test.ts +++ b/packages/grafana-data/src/transformations/transformers/filterByName.test.ts @@ -196,8 +196,83 @@ describe('filterByName transformer', () => { expect(filtered.fields[0].name).toBe('B'); }); }); + it('it can use a variable with multiple comma separated', async () => { + const cfg = { + id: DataTransformerID.filterFieldsByName, + options: { + include: { + variable: '$var', + }, + byVariable: true, + }, + }; - it('uses template variable substituion', async () => { + const ctx = { + interpolate: (target: string | undefined, scopedVars?: ScopedVars, format?: string | Function): string => { + if (!target) { + return ''; + } + const variables: ScopedVars = { + var: { + value: 'B,D', + text: 'Test', + }, + }; + for (const key of Object.keys(variables)) { + return target.replace(`$${key}`, variables[key]!.value); + } + return target; + }, + }; + + await expect(transformDataFrame([cfg], [seriesWithNamesToMatch], ctx)).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields.length).toBe(2); + expect(filtered.fields[0].name).toBe('B'); + expect(filtered.fields[1].name).toBe('D'); + }); + }); + + it('it can use a variable with multiple comma separated values in {}', async () => { + const cfg = { + id: DataTransformerID.filterFieldsByName, + options: { + include: { + variable: '$var', + }, + byVariable: true, + }, + }; + + const ctx = { + interpolate: (target: string | undefined, scopedVars?: ScopedVars, format?: string | Function): string => { + if (!target) { + return ''; + } + const variables: ScopedVars = { + var: { + value: '{B,D}', + text: 'Test', + }, + }; + for (const key of Object.keys(variables)) { + return target.replace(`$${key}`, variables[key]!.value); + } + return target; + }, + }; + + await expect(transformDataFrame([cfg], [seriesWithNamesToMatch], ctx)).toEmitValuesWith((received) => { + const data = received[0]; + const filtered = data[0]; + expect(filtered.fields.length).toBe(2); + expect(filtered.fields[0].name).toBe('B'); + expect(filtered.fields[1].name).toBe('D'); + }); + }); + + it('uses template variable substitution', async () => { const cfg = { id: DataTransformerID.filterFieldsByName, options: { diff --git a/packages/grafana-data/src/transformations/transformers/filterByName.ts b/packages/grafana-data/src/transformations/transformers/filterByName.ts index 1675024636a..0ecc422e866 100644 --- a/packages/grafana-data/src/transformations/transformers/filterByName.ts +++ b/packages/grafana-data/src/transformations/transformers/filterByName.ts @@ -1,4 +1,4 @@ -import { DataTransformerInfo, MatcherConfig } from '../../types/transformations'; +import { DataTransformContext, DataTransformerInfo, MatcherConfig } from '../../types/transformations'; import { FieldMatcherID } from '../matchers/ids'; import { RegexpOrNamesMatcherOptions } from '../matchers/nameMatcher'; @@ -8,6 +8,7 @@ import { DataTransformerID } from './ids'; export interface FilterFieldsByNameTransformerOptions { include?: RegexpOrNamesMatcherOptions; exclude?: RegexpOrNamesMatcherOptions; + byVariable?: boolean; } export const filterFieldsByNameTransformer: DataTransformerInfo = { @@ -20,25 +21,38 @@ export const filterFieldsByNameTransformer: DataTransformerInfo (source) => + operator: (options, ctx) => (source) => source.pipe( filterFieldsTransformer.operator( { - include: getMatcherConfig(options.include), - exclude: getMatcherConfig(options.exclude), + include: getMatcherConfig(ctx, options.include, options.byVariable), + exclude: getMatcherConfig(ctx, options.exclude, options.byVariable), }, - replace + ctx ) ), }; // Exported to share with other implementations, but not exported to `@grafana/data` -export const getMatcherConfig = (options?: RegexpOrNamesMatcherOptions): MatcherConfig | undefined => { +export const getMatcherConfig = ( + ctx: DataTransformContext, + options?: RegexpOrNamesMatcherOptions, + byVariable?: boolean +): MatcherConfig | undefined => { if (!options) { return undefined; } - const { names, pattern } = options; + const { names, pattern, variable } = options; + + if (byVariable && variable) { + const stringOfNames = ctx.interpolate(variable); + if (/\{.*\}/.test(stringOfNames)) { + const namesFromString = stringOfNames.slice(1).slice(0, -1).split(','); + return { id: FieldMatcherID.byNames, options: { names: namesFromString } }; + } + return { id: FieldMatcherID.byNames, options: { names: stringOfNames.split(',') } }; + } if ((!Array.isArray(names) || names.length === 0) && !pattern) { return undefined; diff --git a/public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx b/public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx index 9a3140a1d5e..9c9d675ea58 100644 --- a/public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx +++ b/public/app/features/transformers/editors/FilterByNameTransformerEditor.tsx @@ -1,4 +1,3 @@ -import { css } from '@emotion/css'; import React from 'react'; import { @@ -10,9 +9,11 @@ import { getFieldDisplayName, stringToJsRegex, TransformerCategory, + SelectableValue, } from '@grafana/data'; import { FilterFieldsByNameTransformerOptions } from '@grafana/data/src/transformations/transformers/filterByName'; -import { Field, Input, FilterPill, HorizontalGroup } from '@grafana/ui'; +import { getTemplateSrv } from '@grafana/runtime/src/services'; +import { Input, FilterPill, InlineFieldRow, InlineField, InlineSwitch, Select } from '@grafana/ui'; interface FilterByNameTransformerEditorProps extends TransformerUIProps {} @@ -21,6 +22,9 @@ interface FilterByNameTransformerEditorState { options: FieldNameInfo[]; selected: string[]; regex?: string; + variable?: string; + variables: SelectableValue[]; + byVariable: boolean; isRegexValid?: boolean; } @@ -37,7 +41,10 @@ export class FilterByNameTransformerEditor extends React.PureComponent< this.state = { include: props.options.include?.names || [], regex: props.options.include?.pattern, + variable: props.options.include?.variable, + byVariable: props.options.byVariable || false, options: [], + variables: [], selected: [], isRegexValid: true, }; @@ -57,6 +64,9 @@ export class FilterByNameTransformerEditor extends React.PureComponent< const { input, options } = this.props; const configuredOptions = Array.from(options.include?.names ?? []); + const variables = getTemplateSrv() + .getVariables() + .map((v) => ({ label: '$' + v.name, value: '$' + v.name })); const allNames: FieldNameInfo[] = []; const byName: KeyValue = {}; @@ -97,12 +107,18 @@ export class FilterByNameTransformerEditor extends React.PureComponent< this.setState({ options: allNames, selected: selected.map((s) => s.name), + variables: variables, + byVariable: options.byVariable || false, + variable: options.include?.variable, regex: options.include?.pattern, }); } else { this.setState({ options: allNames, selected: allNames.map((n) => n.name), + variables: variables, + byVariable: options.byVariable || false, + variable: options.include?.variable, regex: options.include?.pattern, }); } @@ -161,19 +177,46 @@ export class FilterByNameTransformerEditor extends React.PureComponent< this.setState({ isRegexValid }); }; + onVariableChange = (selected: SelectableValue) => { + this.props.onChange({ + ...this.props.options, + include: { variable: selected.value }, + }); + + this.setState({ variable: selected.value }); + }; + + onFromVariableChange = (e: React.FormEvent) => { + const val = e.currentTarget.checked; + this.props.onChange({ ...this.props.options, byVariable: val }); + this.setState({ byVariable: val }); + }; + render() { const { options, selected, isRegexValid } = this.state; return ( -
-
-
Identifier
- - + + + + + + {this.state.byVariable ? ( + + + + + + ) : ( + + - + {options.map((o, i) => { const label = `${o.name}${o.count > 1 ? ' (' + o.count + ')' : ''}`; const isSelected = selected.indexOf(o.name) > -1; @@ -197,8 +240,8 @@ export class FilterByNameTransformerEditor extends React.PureComponent< /> ); })} - -
+ + )}
); } diff --git a/public/app/features/transformers/partitionByValues/partitionByValues.ts b/public/app/features/transformers/partitionByValues/partitionByValues.ts index df7a4ecfddf..d33d00f04c7 100644 --- a/public/app/features/transformers/partitionByValues/partitionByValues.ts +++ b/public/app/features/transformers/partitionByValues/partitionByValues.ts @@ -73,7 +73,7 @@ export const partitionByValuesTransformer: SynchronousDataTransformerInfo partitionByValuesTransformer.transformer(options, ctx)(data))), transformer: (options: PartitionByValuesTransformerOptions, ctx: DataTransformContext) => { - const matcherConfig = getMatcherConfig({ names: options.fields }); + const matcherConfig = getMatcherConfig(ctx, { names: options.fields }); if (!matcherConfig) { return noopTransformer.transformer({}, ctx);