diff --git a/devenv/dev-dashboards/transforms/extract-json-paths.json b/devenv/dev-dashboards/transforms/extract-json-paths.json index d709a444680..8b00e21f537 100644 --- a/devenv/dev-dashboards/transforms/extract-json-paths.json +++ b/devenv/dev-dashboards/transforms/extract-json-paths.json @@ -299,7 +299,10 @@ "revision": 1, "schemaVersion": 37, "style": "dark", - "tags": [], + "tags": [ + "gdev", + "transform" + ], "templating": { "list": [] }, @@ -309,7 +312,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Test extractFields JSON", + "title": "Transforms - Test extractFields JSON", "uid": "pD4vPYhVz", "version": 3, "weekStart": "" diff --git a/devenv/dev-dashboards/transforms/filter.json b/devenv/dev-dashboards/transforms/filter.json new file mode 100644 index 00000000000..649514d9261 --- /dev/null +++ b/devenv/dev-dashboards/transforms/filter.json @@ -0,0 +1,171 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1394, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "uid": "PD8C576611E62080A", + "type": "testdata" + }, + "fieldConfig": { + "defaults": { + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "color": { + "mode": "thresholds" + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "targets": [ + { + "scenarioId": "csv_content", + "refId": "A", + "datasource": { + "uid": "PD8C576611E62080A", + "type": "testdata" + }, + "csvContent": "AAA\n1\n2\n3\n4" + }, + { + "scenarioId": "csv_content", + "refId": "B", + "datasource": { + "uid": "PD8C576611E62080A", + "type": "testdata" + }, + "csvContent": "BBB\n1\n2\n3\n4\n", + "hide": false + } + ], + "title": "Transformer query filters", + "type": "table", + "transformations": [ + { + "id": "reduce", + "options": { + "reducers": [ + "min" + ], + "mode": "reduceFields", + "includeTimeField": false + }, + "filter": { + "id": "byRefId", + "options": "A" + } + }, + { + "id": "reduce", + "options": { + "reducers": [ + "max" + ], + "mode": "reduceFields", + "includeTimeField": false + }, + "filter": { + "id": "byRefId", + "options": "B" + } + }, + { + "id": "concatenate", + "options": {} + }, + { + "id": "organize", + "options": { + "excludeByName": {}, + "indexByName": {}, + "renameByName": { + "AAA": "Min from Query A", + "BBB": "Max from Query B" + } + } + } + ], + "options": { + "showHeader": true, + "footer": { + "show": false, + "reducer": [ + "sum" + ], + "countRows": false, + "fields": "" + }, + "frameIndex": 0 + }, + "pluginVersion": "9.4.0-pre" + } + ], + "schemaVersion": 37, + "style": "dark", + "tags": [ + "gdev", + "transform" + ], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Transforms - Filters", + "uid": "fGWBVW4k" + } diff --git a/devenv/dev-dashboards/transforms/join-by-field.json b/devenv/dev-dashboards/transforms/join-by-field.json index a86b3b89a89..19af33017b9 100644 --- a/devenv/dev-dashboards/transforms/join-by-field.json +++ b/devenv/dev-dashboards/transforms/join-by-field.json @@ -624,7 +624,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Join by field", + "title": "Transforms - Join by field", "uid": "gw0K4rmVz", "version": 6, "weekStart": "" diff --git a/devenv/dev-dashboards/transforms/join-by-labels.json b/devenv/dev-dashboards/transforms/join-by-labels.json index 951a670f405..b9d37e554a5 100644 --- a/devenv/dev-dashboards/transforms/join-by-labels.json +++ b/devenv/dev-dashboards/transforms/join-by-labels.json @@ -347,7 +347,7 @@ }, "timepicker": {}, "timezone": "", - "title": "Join by labels", + "title": "Transforms - Join by labels", "uid": "FVl-9CR4z", "version": 10, "weekStart": "" diff --git a/devenv/dev-dashboards/transforms/reuse.json b/devenv/dev-dashboards/transforms/reuse.json index 41667ce6933..f3e7bedc5bc 100644 --- a/devenv/dev-dashboards/transforms/reuse.json +++ b/devenv/dev-dashboards/transforms/reuse.json @@ -521,7 +521,10 @@ ], "schemaVersion": 37, "style": "dark", - "tags": ["devenv"], + "tags": [ + "gdev", + "transform" + ], "templating": { "list": [] }, @@ -531,6 +534,6 @@ }, "timepicker": {}, "timezone": "", - "title": "Reuse dashboard queries", + "title": "Transforms - Reuse dashboard queries", "uid": "fYGWTVW4k" } diff --git a/devenv/jsonnet/dev-dashboards.libsonnet b/devenv/jsonnet/dev-dashboards.libsonnet index a8ec282659e..6591cc019ce 100644 --- a/devenv/jsonnet/dev-dashboards.libsonnet +++ b/devenv/jsonnet/dev-dashboards.libsonnet @@ -184,6 +184,13 @@ local dashboard = grafana.dashboard; id: 0, } }, + dashboard.new('filter', import '../dev-dashboards/transforms/filter.json') + + resource.addMetadata('folder', 'dev-dashboards') + + { + spec+: { + id: 0, + } + }, dashboard.new('gauge-multi-series', import '../dev-dashboards/panel-gauge/gauge-multi-series.json') + resource.addMetadata('folder', 'dev-dashboards') + { diff --git a/kinds/dashboard/dashboard_kind.cue b/kinds/dashboard/dashboard_kind.cue index 002331eb31b..98ed1bb3f02 100644 --- a/kinds/dashboard/dashboard_kind.cue +++ b/kinds/dashboard/dashboard_kind.cue @@ -285,7 +285,10 @@ lineage: seqs: [ // TODO docs // FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it #Transformation: { - id: string + id: string + hide: bool | *false + // only apply to some frames + filter?: #MatcherConfig options: {...} } @cuetsy(kind="interface") @grafanamaturity(NeedsExpertReview) diff --git a/packages/grafana-data/src/transformations/matchers/ids.ts b/packages/grafana-data/src/transformations/matchers/ids.ts index cd4c7888bd5..f833d34db2a 100644 --- a/packages/grafana-data/src/transformations/matchers/ids.ts +++ b/packages/grafana-data/src/transformations/matchers/ids.ts @@ -35,7 +35,6 @@ export enum FrameMatcherID { byName = 'byName', byRefId = 'byRefId', byIndex = 'byIndex', - byLabel = 'byLabel', } /** diff --git a/packages/grafana-data/src/transformations/transformDataFrame.test.ts b/packages/grafana-data/src/transformations/transformDataFrame.test.ts index 08412b09b68..61e1a48488a 100644 --- a/packages/grafana-data/src/transformations/transformDataFrame.test.ts +++ b/packages/grafana-data/src/transformations/transformDataFrame.test.ts @@ -3,10 +3,11 @@ import { FieldType } from '../types'; import { mockTransformationsRegistry } from '../utils/tests/mockTransformationsRegistry'; import { ReducerID } from './fieldReducer'; +import { FrameMatcherID } from './matchers/ids'; import { transformDataFrame } from './transformDataFrame'; import { filterFieldsByNameTransformer } from './transformers/filterByName'; import { DataTransformerID } from './transformers/ids'; -import { reduceTransformer } from './transformers/reduce'; +import { reduceTransformer, ReduceTransformerMode } from './transformers/reduce'; const seriesAWithSingleField = toDataFrame({ name: 'A', @@ -73,4 +74,44 @@ describe('transformDataFrame', () => { expect(processed[0].fields[0].values.get(0)).toEqual('temperature'); }); }); + + it('Support filtering', async () => { + const frameA = toDataFrame({ + refId: 'A', + fields: [{ name: 'value', type: FieldType.number, values: [5, 6] }], + }); + const frameB = toDataFrame({ + refId: 'B', + fields: [{ name: 'value', type: FieldType.number, values: [7, 8] }], + }); + + const cfg = [ + { + id: DataTransformerID.reduce, + filter: { + id: FrameMatcherID.byRefId, + options: 'A', // Only apply to A + }, + options: { + reducers: [ReducerID.first], + mode: ReduceTransformerMode.ReduceFields, + }, + }, + ]; + + // Only apply A + await expect(transformDataFrame(cfg, [frameA, frameB])).toEmitValuesWith((received) => { + const processed = received[0].map((v) => v.fields[0].values.toArray()); + expect(processed).toBeTruthy(); + expect(processed).toMatchObject([[5], [7, 8]]); + }); + + // Only apply to B + cfg[0].filter.options = 'B'; + await expect(transformDataFrame(cfg, [frameA, frameB])).toEmitValuesWith((received) => { + const processed = received[0].map((v) => v.fields[0].values.toArray()); + expect(processed).toBeTruthy(); + expect(processed).toMatchObject([[5, 6], [7]]); + }); + }); }); diff --git a/packages/grafana-data/src/transformations/transformDataFrame.ts b/packages/grafana-data/src/transformations/transformDataFrame.ts index 2c2e6e1350e..fbd6951013e 100644 --- a/packages/grafana-data/src/transformations/transformDataFrame.ts +++ b/packages/grafana-data/src/transformations/transformDataFrame.ts @@ -1,8 +1,9 @@ import { MonoTypeOperatorFunction, Observable, of } from 'rxjs'; import { map, mergeMap } from 'rxjs/operators'; -import { DataFrame, DataTransformContext, DataTransformerConfig } from '../types'; +import { DataFrame, DataTransformContext, DataTransformerConfig, FrameMatcher } from '../types'; +import { getFrameMatchers } from './matchers'; import { standardTransformersRegistry, TransformerRegistryItem } from './standardTransformersRegistry'; const getOperator = @@ -17,15 +18,30 @@ const getOperator = const defaultOptions = info.transformation.defaultOptions ?? {}; const options = { ...defaultOptions, ...config.options }; + const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined; return source.pipe( mergeMap((before) => - of(before).pipe(info.transformation.operator(options, ctx), postProcessTransform(before, info)) + of(filterInput(before, matcher)).pipe( + info.transformation.operator(options, ctx), + postProcessTransform(before, info, matcher) + ) ) ); }; +function filterInput(data: DataFrame[], matcher?: FrameMatcher) { + if (matcher) { + return data.filter((v) => matcher(v)); + } + return data; +} + const postProcessTransform = - (before: DataFrame[], info: TransformerRegistryItem): MonoTypeOperatorFunction => + ( + before: DataFrame[], + info: TransformerRegistryItem, + matcher?: FrameMatcher + ): MonoTypeOperatorFunction => (source) => source.pipe( map((after) => { @@ -46,6 +62,21 @@ const postProcessTransform = } } + // Add back the filtered out frames + if (matcher) { + // keep the frame order the same + let insert = 0; + const append = before.filter((v, idx) => { + const keep = !matcher(v); + if (keep && !insert) { + insert = idx; + } + return keep; + }); + if (append.length) { + after.splice(insert, 0, ...append); + } + } return after; }) ); diff --git a/packages/grafana-data/src/types/transformations.ts b/packages/grafana-data/src/types/transformations.ts index 270ec1cbdda..88968acc864 100644 --- a/packages/grafana-data/src/types/transformations.ts +++ b/packages/grafana-data/src/types/transformations.ts @@ -1,11 +1,15 @@ -export type { MatcherConfig } from '@grafana/schema'; import { MonoTypeOperatorFunction } from 'rxjs'; +import { MatcherConfig } from '@grafana/schema'; + import { RegistryItemWithOptions } from '../utils/Registry'; import { DataFrame, Field } from './dataFrame'; import { InterpolateFunction } from './panel'; +/** deprecated, use it from schema */ +export type { MatcherConfig }; + /** * Context passed to transformDataFrame and to each transform operator */ @@ -44,10 +48,15 @@ export interface DataTransformerConfig { * Unique identifier of transformer */ id: string; + /** * Disabled transformations are skipped */ disabled?: boolean; + + /** Optional frame matcher. When missing it will be applied to all results */ + filter?: MatcherConfig; + /** * Options to be passed to the transformer */ diff --git a/packages/grafana-schema/src/index.gen.ts b/packages/grafana-schema/src/index.gen.ts index f37d525a6ad..1368bee47b1 100644 --- a/packages/grafana-schema/src/index.gen.ts +++ b/packages/grafana-schema/src/index.gen.ts @@ -44,6 +44,7 @@ export { defaultThresholdsConfig, MappingType, SpecialValueMatch, + defaultTransformation, DashboardCursorSync, defaultDashboardCursorSync, defaultRowPanel diff --git a/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts b/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts index 95b22b152ce..b7d2041a0e9 100644 --- a/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts +++ b/packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts @@ -356,10 +356,19 @@ export interface ValueMappingResult { * FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it */ export interface Transformation { + /** + * only apply to some frames + */ + filter?: MatcherConfig; + hide: boolean; id: string; options: Record; } +export const defaultTransformation: Partial = { + hide: false, +}; + /** * 0 for no shared crosshair or tooltip (default). * 1 for shared crosshair. diff --git a/pkg/kinds/dashboard/dashboard_types_gen.go b/pkg/kinds/dashboard/dashboard_types_gen.go index 9fdc6e22532..6c51c2af578 100644 --- a/pkg/kinds/dashboard/dashboard_types_gen.go +++ b/pkg/kinds/dashboard/dashboard_types_gen.go @@ -710,6 +710,8 @@ type ThresholdsMode string // TODO docs // FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it type Transformation struct { + Filter *MatcherConfig `json:"filter,omitempty"` + Hide bool `json:"hide"` Id string `json:"id"` Options map[string]interface{} `json:"options"` } diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx index c6644031ade..26fe38488a2 100644 --- a/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx @@ -8,6 +8,7 @@ import { GrafanaTheme2, transformDataFrame, TransformerRegistryItem, + getFrameMatchers, } from '@grafana/data'; import { selectors } from '@grafana/e2e-selectors'; import { Icon, JSONFormatter, useStyles2 } from '@grafana/ui'; @@ -37,9 +38,16 @@ export const TransformationEditor = ({ const config = useMemo(() => configs[index], [configs, index]); useEffect(() => { + const config = configs[index].transformation; + const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined; const inputTransforms = configs.slice(0, index).map((t) => t.transformation); const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation); - const inputSubscription = transformDataFrame(inputTransforms, data).subscribe(setInput); + const inputSubscription = transformDataFrame(inputTransforms, data).subscribe((v) => { + if (matcher) { + v = data.filter((v) => matcher(v)); + } + setInput(v); + }); const outputSubscription = transformDataFrame(inputTransforms, data) .pipe(mergeMap((before) => transformDataFrame(outputTransforms, before))) .subscribe(setOutput); @@ -56,18 +64,13 @@ export const TransformationEditor = ({ options: { ...uiConfig.transformation.defaultOptions, ...config.transformation.options }, input, onChange: (opts) => { - onChange(index, { id: config.transformation.id, options: opts }); + onChange(index, { + ...config.transformation, + options: opts, + }); }, }), - [ - uiConfig.editor, - uiConfig.transformation.defaultOptions, - config.transformation.options, - config.transformation.id, - input, - onChange, - index, - ] + [uiConfig.editor, uiConfig.transformation.defaultOptions, config.transformation, input, onChange, index] ); return ( diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx new file mode 100644 index 00000000000..ebbccbab512 --- /dev/null +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx @@ -0,0 +1,55 @@ +import { css } from '@emotion/css'; +import React, { useMemo } from 'react'; + +import { + DataFrame, + DataTransformerConfig, + GrafanaTheme2, + StandardEditorContext, + StandardEditorsRegistryItem, +} from '@grafana/data'; +import { useStyles2 } from '@grafana/ui'; +import { FrameSelectionEditor } from 'app/plugins/panel/geomap/editor/FrameSelectionEditor'; + +interface TransformationFilterProps { + index: number; + config: DataTransformerConfig; + data: DataFrame[]; + onChange: (index: number, config: DataTransformerConfig) => void; +} + +export const TransformationFilter = ({ index, data, config, onChange }: TransformationFilterProps) => { + const styles = useStyles2(getStyles); + const context = useMemo(() => { + // eslint-disable-next-line + return { data } as StandardEditorContext; + }, [data]); + + return ( +
+
Apply tranformation to
+ onChange(index, { ...config, filter })} + /> +
+ ); +}; + +const getStyles = (theme: GrafanaTheme2) => { + const borderRadius = theme.shape.borderRadius(); + + return { + wrapper: css` + padding: ${theme.spacing(2)}; + border: 2px solid ${theme.colors.background.secondary}; + border-top: none; + border-radius: 0 0 ${borderRadius} ${borderRadius}; + position: relative; + top: -4px; + `, + }; +}; diff --git a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx index 71eeb0200e1..b56f09d8a6d 100644 --- a/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx +++ b/public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx @@ -1,7 +1,7 @@ import React, { useCallback } from 'react'; import { useToggle } from 'react-use'; -import { DataFrame, DataTransformerConfig, TransformerRegistryItem } from '@grafana/data'; +import { DataFrame, DataTransformerConfig, TransformerRegistryItem, FrameMatcherID } from '@grafana/data'; import { HorizontalGroup } from '@grafana/ui'; import { OperationRowHelp } from 'app/core/components/QueryOperationRow/OperationRowHelp'; import { QueryOperationAction } from 'app/core/components/QueryOperationRow/QueryOperationAction'; @@ -12,6 +12,7 @@ import { import { PluginStateInfo } from 'app/features/plugins/components/PluginStateInfo'; import { TransformationEditor } from './TransformationEditor'; +import { TransformationFilter } from './TransformationFilter'; import { TransformationsEditorTransformation } from './types'; interface TransformationOperationRowProps { @@ -24,7 +25,7 @@ interface TransformationOperationRowProps { onChange: (index: number, config: DataTransformerConfig) => void; } -export const TransformationOperationRow: React.FC = ({ +export const TransformationOperationRow = ({ onRemove, index, id, @@ -32,10 +33,12 @@ export const TransformationOperationRow: React.FC { +}: TransformationOperationRowProps) => { const [showDebug, toggleDebug] = useToggle(false); const [showHelp, toggleHelp] = useToggle(false); const disabled = configs[index].transformation.disabled; + const filter = configs[index].transformation.filter != null; + const showFilter = filter || data.length > 1; const onDisableToggle = useCallback( (index: number) => { @@ -48,6 +51,20 @@ export const TransformationOperationRow: React.FC { + let current = { ...configs[index].transformation }; + if (current.filter) { + delete current.filter; + } else { + current.filter = { + id: FrameMatcherID.byRefId, + options: '', // empty string will not do anything + }; + } + onChange(index, current); + }, [onChange, index, configs]); + const renderActions = ({ isOpen }: QueryOperationRowRenderProps) => { return ( @@ -58,6 +75,7 @@ export const TransformationOperationRow: React.FC + {showFilter && } {showHelp && } + {filter && ( + + )} {} @@ -103,28 +103,36 @@ export class FilterByRefIdTransformerEditor extends React.PureComponent< render() { const { options, selected } = this.state; + const { input } = this.props; return ( -
-
-
Series refId
- - {options.map((o, i) => { - const label = `${o.refId}${o.count > 1 ? ' (' + o.count + ')' : ''}`; - const isSelected = selected.indexOf(o.refId) > -1; - return ( - { - this.onFieldToggle(o.refId); - }} - label={label} - selected={isSelected} - /> - ); - })} - + <> + {input.length <= 1 && ( +
+ Filter data by query expects multiple queries in the input. +
+ )} +
+
+
Series refId
+ + {options.map((o, i) => { + const label = `${o.refId}${o.count > 1 ? ' (' + o.count + ')' : ''}`; + const isSelected = selected.indexOf(o.refId) > -1; + return ( + { + this.onFieldToggle(o.refId); + }} + label={label} + selected={isSelected} + /> + ); + })} + +
-
+ ); } } diff --git a/public/app/features/transformers/editors/OrganizeFieldsTransformerEditor.tsx b/public/app/features/transformers/editors/OrganizeFieldsTransformerEditor.tsx index 72b6a5312b5..dadcb2b61f5 100644 --- a/public/app/features/transformers/editors/OrganizeFieldsTransformerEditor.tsx +++ b/public/app/features/transformers/editors/OrganizeFieldsTransformerEditor.tsx @@ -17,8 +17,7 @@ import { useAllFieldNamesFromDataFrames } from '../utils'; interface OrganizeFieldsTransformerEditorProps extends TransformerUIProps {} -const OrganizeFieldsTransformerEditor: React.FC = (props) => { - const { options, input, onChange } = props; +const OrganizeFieldsTransformerEditor = ({ options, input, onChange }: OrganizeFieldsTransformerEditorProps) => { const { indexByName, excludeByName, renameByName } = options; const fieldNames = useAllFieldNamesFromDataFrames(input); @@ -75,7 +74,8 @@ const OrganizeFieldsTransformerEditor: React.FC 1) { return ( - Organize fields only works with a single frame. Consider applying a join transformation first. + Organize fields only works with a single frame. Consider applying a join transformation or filtering the input + first. ); }