IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-11-22 08:56:43 -06:00
Code Issues Packages Projects Releases Wiki Activity

Transformations: Selectively apply transformation to queries (#61735)

Browse Source
This commit is contained in:
Ryan McKinley 2023-01-31 09:06:06 -08:00 committed by GitHub
parent e7bfc4e749
commit bba80b6c7a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 417 additions and 51 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
devenv/dev-dashboards/transforms/extract-json-paths.json
View File

@ -299,7 +299,10 @@
"revision": 1,
"schemaVersion": 37,
"style": "dark",
"tags": [],
"tags": [
"gdev",
"transform"
],
"templating": {
"list": []
},
@ -309,7 +312,7 @@
},
"timepicker": {},
"timezone": "",
"title": "Test extractFields JSON",
"title": "Transforms - Test extractFields JSON",
"uid": "pD4vPYhVz",
"version": 3,
"weekStart": ""

171
devenv/dev-dashboards/transforms/filter.json Normal file
View File

@ -0,0 +1,171 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"target": {
"limit": 100,
"matchAny": false,
"tags": [],
"type": "dashboard"
},
"type": "dashboard"
}
]
},
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": 1394,
"links": [],
"liveNow": false,
"panels": [
{
"datasource": {
"uid": "PD8C576611E62080A",
"type": "testdata"
},
"fieldConfig": {
"defaults": {
"custom": {
"align": "auto",
"cellOptions": {
"type": "auto"
},
"inspect": false
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green"
},
{
"color": "red",
"value": 80
}
]
},
"color": {
"mode": "thresholds"
}
},
"overrides": []
},
"gridPos": {
"h": 9,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"targets": [
{
"scenarioId": "csv_content",
"refId": "A",
"datasource": {
"uid": "PD8C576611E62080A",
"type": "testdata"
},
"csvContent": "AAA\n1\n2\n3\n4"
},
{
"scenarioId": "csv_content",
"refId": "B",
"datasource": {
"uid": "PD8C576611E62080A",
"type": "testdata"
},
"csvContent": "BBB\n1\n2\n3\n4\n",
"hide": false
}
],
"title": "Transformer query filters",
"type": "table",
"transformations": [
{
"id": "reduce",
"options": {
"reducers": [
"min"
],
"mode": "reduceFields",
"includeTimeField": false
},
"filter": {
"id": "byRefId",
"options": "A"
}
},
{
"id": "reduce",
"options": {
"reducers": [
"max"
],
"mode": "reduceFields",
"includeTimeField": false
},
"filter": {
"id": "byRefId",
"options": "B"
}
},
{
"id": "concatenate",
"options": {}
},
{
"id": "organize",
"options": {
"excludeByName": {},
"indexByName": {},
"renameByName": {
"AAA": "Min from Query A",
"BBB": "Max from Query B"
}
}
}
],
"options": {
"showHeader": true,
"footer": {
"show": false,
"reducer": [
"sum"
],
"countRows": false,
"fields": ""
},
"frameIndex": 0
},
"pluginVersion": "9.4.0-pre"
}
],
"schemaVersion": 37,
"style": "dark",
"tags": [
"gdev",
"transform"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
},
"timepicker": {},
"timezone": "",
"title": "Transforms - Filters",
"uid": "fGWBVW4k"
}

2
devenv/dev-dashboards/transforms/join-by-field.json
View File

@ -624,7 +624,7 @@
},
"timepicker": {},
"timezone": "",
"title": "Join by field",
"title": "Transforms - Join by field",
"uid": "gw0K4rmVz",
"version": 6,
"weekStart": ""

2
devenv/dev-dashboards/transforms/join-by-labels.json
View File

@ -347,7 +347,7 @@
},
"timepicker": {},
"timezone": "",
"title": "Join by labels",
"title": "Transforms - Join by labels",
"uid": "FVl-9CR4z",
"version": 10,
"weekStart": ""

7
devenv/dev-dashboards/transforms/reuse.json
View File

@ -521,7 +521,10 @@
],
"schemaVersion": 37,
"style": "dark",
"tags": ["devenv"],
"tags": [
"gdev",
"transform"
],
"templating": {
"list": []
},
@ -531,6 +534,6 @@
},
"timepicker": {},
"timezone": "",
"title": "Reuse dashboard queries",
"title": "Transforms - Reuse dashboard queries",
"uid": "fYGWTVW4k"
}

7
devenv/jsonnet/dev-dashboards.libsonnet
View File

@ -184,6 +184,13 @@ local dashboard = grafana.dashboard;
id: 0,
}
},
dashboard.new('filter', import '../dev-dashboards/transforms/filter.json') +
resource.addMetadata('folder', 'dev-dashboards') +
{
spec+: {
id: 0,
}
},
dashboard.new('gauge-multi-series', import '../dev-dashboards/panel-gauge/gauge-multi-series.json') +
resource.addMetadata('folder', 'dev-dashboards') +
{

3
kinds/dashboard/dashboard_kind.cue
View File

@ -286,6 +286,9 @@ lineage: seqs: [
// FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it
#Transformation: {
id: string
hide: bool | *false
// only apply to some frames
filter?: #MatcherConfig
options: {...}
} @cuetsy(kind="interface") @grafanamaturity(NeedsExpertReview)

1
packages/grafana-data/src/transformations/matchers/ids.ts
View File

@ -35,7 +35,6 @@ export enum FrameMatcherID {
byName = 'byName',
byRefId = 'byRefId',
byIndex = 'byIndex',
byLabel = 'byLabel',
}
/**

43
packages/grafana-data/src/transformations/transformDataFrame.test.ts
View File

@ -3,10 +3,11 @@ import { FieldType } from '../types';
import { mockTransformationsRegistry } from '../utils/tests/mockTransformationsRegistry';
import { ReducerID } from './fieldReducer';
import { FrameMatcherID } from './matchers/ids';
import { transformDataFrame } from './transformDataFrame';
import { filterFieldsByNameTransformer } from './transformers/filterByName';
import { DataTransformerID } from './transformers/ids';
import { reduceTransformer } from './transformers/reduce';
import { reduceTransformer, ReduceTransformerMode } from './transformers/reduce';
const seriesAWithSingleField = toDataFrame({
name: 'A',
@ -73,4 +74,44 @@ describe('transformDataFrame', () => {
expect(processed[0].fields[0].values.get(0)).toEqual('temperature');
});
});
it('Support filtering', async () => {
const frameA = toDataFrame({
refId: 'A',
fields: [{ name: 'value', type: FieldType.number, values: [5, 6] }],
});
const frameB = toDataFrame({
refId: 'B',
fields: [{ name: 'value', type: FieldType.number, values: [7, 8] }],
});
const cfg = [
{
id: DataTransformerID.reduce,
filter: {
id: FrameMatcherID.byRefId,
options: 'A', // Only apply to A
},
options: {
reducers: [ReducerID.first],
mode: ReduceTransformerMode.ReduceFields,
},
},
];
// Only apply A
await expect(transformDataFrame(cfg, [frameA, frameB])).toEmitValuesWith((received) => {
const processed = received[0].map((v) => v.fields[0].values.toArray());
expect(processed).toBeTruthy();
expect(processed).toMatchObject([[5], [7, 8]]);
});
// Only apply to B
cfg[0].filter.options = 'B';
await expect(transformDataFrame(cfg, [frameA, frameB])).toEmitValuesWith((received) => {
const processed = received[0].map((v) => v.fields[0].values.toArray());
expect(processed).toBeTruthy();
expect(processed).toMatchObject([[5, 6], [7]]);
});
});
});

37
packages/grafana-data/src/transformations/transformDataFrame.ts
View File

@ -1,8 +1,9 @@
import { MonoTypeOperatorFunction, Observable, of } from 'rxjs';
import { map, mergeMap } from 'rxjs/operators';
import { DataFrame, DataTransformContext, DataTransformerConfig } from '../types';
import { DataFrame, DataTransformContext, DataTransformerConfig, FrameMatcher } from '../types';
import { getFrameMatchers } from './matchers';
import { standardTransformersRegistry, TransformerRegistryItem } from './standardTransformersRegistry';
const getOperator =
@ -17,15 +18,30 @@ const getOperator =
const defaultOptions = info.transformation.defaultOptions ?? {};
const options = { ...defaultOptions, ...config.options };
const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
return source.pipe(
mergeMap((before) =>
of(before).pipe(info.transformation.operator(options, ctx), postProcessTransform(before, info))
of(filterInput(before, matcher)).pipe(
info.transformation.operator(options, ctx),
postProcessTransform(before, info, matcher)
)
)
);
};
function filterInput(data: DataFrame[], matcher?: FrameMatcher) {
if (matcher) {
return data.filter((v) => matcher(v));
}
return data;
}
const postProcessTransform =
(before: DataFrame[], info: TransformerRegistryItem<any>): MonoTypeOperatorFunction<DataFrame[]> =>
(
before: DataFrame[],
info: TransformerRegistryItem<any>,
matcher?: FrameMatcher
): MonoTypeOperatorFunction<DataFrame[]> =>
(source) =>
source.pipe(
map((after) => {
@ -46,6 +62,21 @@ const postProcessTransform =
}
}
// Add back the filtered out frames
if (matcher) {
// keep the frame order the same
let insert = 0;
const append = before.filter((v, idx) => {
const keep = !matcher(v);
if (keep && !insert) {
insert = idx;
}
return keep;
});
if (append.length) {
after.splice(insert, 0, ...append);
}
}
return after;
})
);

11
packages/grafana-data/src/types/transformations.ts
View File

@ -1,11 +1,15 @@
export type { MatcherConfig } from '@grafana/schema';
import { MonoTypeOperatorFunction } from 'rxjs';
import { MatcherConfig } from '@grafana/schema';
import { RegistryItemWithOptions } from '../utils/Registry';
import { DataFrame, Field } from './dataFrame';
import { InterpolateFunction } from './panel';
/** deprecated, use it from schema */
export type { MatcherConfig };
/**
* Context passed to transformDataFrame and to each transform operator
*/
@ -44,10 +48,15 @@ export interface DataTransformerConfig<TOptions = any> {
* Unique identifier of transformer
*/
id: string;
/**
* Disabled transformations are skipped
*/
disabled?: boolean;
/** Optional frame matcher. When missing it will be applied to all results */
filter?: MatcherConfig;
/**
* Options to be passed to the transformer
*/

1
packages/grafana-schema/src/index.gen.ts
View File

@ -44,6 +44,7 @@ export {
defaultThresholdsConfig,
MappingType,
SpecialValueMatch,
defaultTransformation,
DashboardCursorSync,
defaultDashboardCursorSync,
defaultRowPanel

9
packages/grafana-schema/src/raw/dashboard/x/dashboard_types.gen.ts
View File

@ -356,10 +356,19 @@ export interface ValueMappingResult {
* FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it
*/
export interface Transformation {
/**
* only apply to some frames
*/
filter?: MatcherConfig;
hide: boolean;
id: string;
options: Record<string, unknown>;
}
export const defaultTransformation: Partial<Transformation> = {
hide: false,
};
/**
* 0 for no shared crosshair or tooltip (default).
* 1 for shared crosshair.

2
pkg/kinds/dashboard/dashboard_types_gen.go
View File

@ -710,6 +710,8 @@ type ThresholdsMode string
// TODO docs
// FIXME this is extremely underspecfied; wasn't obvious which typescript types corresponded to it
type Transformation struct {
Filter *MatcherConfig `json:"filter,omitempty"`
Hide bool `json:"hide"`
Id string `json:"id"`
Options map[string]interface{} `json:"options"`
}

25
public/app/features/dashboard/components/TransformationsEditor/TransformationEditor.tsx
View File

@ -8,6 +8,7 @@ import {
GrafanaTheme2,
transformDataFrame,
TransformerRegistryItem,
getFrameMatchers,
} from '@grafana/data';
import { selectors } from '@grafana/e2e-selectors';
import { Icon, JSONFormatter, useStyles2 } from '@grafana/ui';
@ -37,9 +38,16 @@ export const TransformationEditor = ({
const config = useMemo(() => configs[index], [configs, index]);
useEffect(() => {
const config = configs[index].transformation;
const matcher = config.filter?.options ? getFrameMatchers(config.filter) : undefined;
const inputTransforms = configs.slice(0, index).map((t) => t.transformation);
const outputTransforms = configs.slice(index, index + 1).map((t) => t.transformation);
const inputSubscription = transformDataFrame(inputTransforms, data).subscribe(setInput);
const inputSubscription = transformDataFrame(inputTransforms, data).subscribe((v) => {
if (matcher) {
v = data.filter((v) => matcher(v));
}
setInput(v);
});
const outputSubscription = transformDataFrame(inputTransforms, data)
.pipe(mergeMap((before) => transformDataFrame(outputTransforms, before)))
.subscribe(setOutput);
@ -56,18 +64,13 @@ export const TransformationEditor = ({
options: { ...uiConfig.transformation.defaultOptions, ...config.transformation.options },
input,
onChange: (opts) => {
onChange(index, { id: config.transformation.id, options: opts });
onChange(index, {
...config.transformation,
options: opts,
});
},
}),
[
uiConfig.editor,
uiConfig.transformation.defaultOptions,
config.transformation.options,
config.transformation.id,
input,
onChange,
index,
]
[uiConfig.editor, uiConfig.transformation.defaultOptions, config.transformation, input, onChange, index]
);
return (

55
public/app/features/dashboard/components/TransformationsEditor/TransformationFilter.tsx Normal file
View File

@ -0,0 +1,55 @@
import { css } from '@emotion/css';
import React, { useMemo } from 'react';
import {
DataFrame,
DataTransformerConfig,
GrafanaTheme2,
StandardEditorContext,
StandardEditorsRegistryItem,
} from '@grafana/data';
import { useStyles2 } from '@grafana/ui';
import { FrameSelectionEditor } from 'app/plugins/panel/geomap/editor/FrameSelectionEditor';
interface TransformationFilterProps {
index: number;
config: DataTransformerConfig;
data: DataFrame[];
onChange: (index: number, config: DataTransformerConfig) => void;
}
export const TransformationFilter = ({ index, data, config, onChange }: TransformationFilterProps) => {
const styles = useStyles2(getStyles);
const context = useMemo(() => {
// eslint-disable-next-line
return { data } as StandardEditorContext<unknown>;
}, [data]);
return (
<div className={styles.wrapper}>
<h5>Apply tranformation to</h5>
<FrameSelectionEditor
value={config.filter!}
context={context}
// eslint-disable-next-line
item={{} as StandardEditorsRegistryItem}
onChange={(filter) => onChange(index, { ...config, filter })}
/>
</div>
);
};
const getStyles = (theme: GrafanaTheme2) => {
const borderRadius = theme.shape.borderRadius();
return {
wrapper: css`
padding: ${theme.spacing(2)};
border: 2px solid ${theme.colors.background.secondary};
border-top: none;
border-radius: 0 0 ${borderRadius} ${borderRadius};
position: relative;
top: -4px;
`,
};
};

27
public/app/features/dashboard/components/TransformationsEditor/TransformationOperationRow.tsx
View File

@ -1,7 +1,7 @@
import React, { useCallback } from 'react';
import { useToggle } from 'react-use';
import { DataFrame, DataTransformerConfig, TransformerRegistryItem } from '@grafana/data';
import { DataFrame, DataTransformerConfig, TransformerRegistryItem, FrameMatcherID } from '@grafana/data';
import { HorizontalGroup } from '@grafana/ui';
import { OperationRowHelp } from 'app/core/components/QueryOperationRow/OperationRowHelp';
import { QueryOperationAction } from 'app/core/components/QueryOperationRow/QueryOperationAction';
@ -12,6 +12,7 @@ import {
import { PluginStateInfo } from 'app/features/plugins/components/PluginStateInfo';
import { TransformationEditor } from './TransformationEditor';
import { TransformationFilter } from './TransformationFilter';
import { TransformationsEditorTransformation } from './types';
interface TransformationOperationRowProps {
@ -24,7 +25,7 @@ interface TransformationOperationRowProps {
onChange: (index: number, config: DataTransformerConfig) => void;
}
export const TransformationOperationRow: React.FC<TransformationOperationRowProps> = ({
export const TransformationOperationRow = ({
onRemove,
index,
id,
@ -32,10 +33,12 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
configs,
uiConfig,
onChange,
}) => {
}: TransformationOperationRowProps) => {
const [showDebug, toggleDebug] = useToggle(false);
const [showHelp, toggleHelp] = useToggle(false);
const disabled = configs[index].transformation.disabled;
const filter = configs[index].transformation.filter != null;
const showFilter = filter || data.length > 1;
const onDisableToggle = useCallback(
(index: number) => {
@ -48,6 +51,20 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
[onChange, configs]
);
// Adds or removes the frame filter
const toggleFilter = useCallback(() => {
let current = { ...configs[index].transformation };
if (current.filter) {
delete current.filter;
} else {
current.filter = {
id: FrameMatcherID.byRefId,
options: '', // empty string will not do anything
};
}
onChange(index, current);
}, [onChange, index, configs]);
const renderActions = ({ isOpen }: QueryOperationRowRenderProps) => {
return (
<HorizontalGroup align="center" width="auto">
@ -58,6 +75,7 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
onClick={toggleHelp}
active={showHelp}
/>
{showFilter && <QueryOperationAction title="Filter" icon="filter" onClick={toggleFilter} active={filter} />}
<QueryOperationAction title="Debug" disabled={!isOpen} icon="bug" onClick={toggleDebug} active={showDebug} />
<QueryOperationAction
title="Disable/Enable transformation"
@ -80,6 +98,9 @@ export const TransformationOperationRow: React.FC<TransformationOperationRowProp
disabled={disabled}
>
{showHelp && <OperationRowHelp markdown={prepMarkdown(uiConfig)} />}
{filter && (
<TransformationFilter index={index} config={configs[index].transformation} data={data} onChange={onChange} />
)}
<TransformationEditor
debugMode={showDebug}
index={index}

10
public/app/features/transformers/editors/FilterByRefIdTransformerEditor.tsx
View File

@ -8,7 +8,7 @@ import {
TransformerUIProps,
} from '@grafana/data';
import { FilterFramesByRefIdTransformerOptions } from '@grafana/data/src/transformations/transformers/filterByRefId';
import { HorizontalGroup, FilterPill } from '@grafana/ui';
import { HorizontalGroup, FilterPill, FieldValidationMessage } from '@grafana/ui';
interface FilterByRefIdTransformerEditorProps extends TransformerUIProps<FilterFramesByRefIdTransformerOptions> {}
@ -103,7 +103,14 @@ export class FilterByRefIdTransformerEditor extends React.PureComponent<
render() {
const { options, selected } = this.state;
const { input } = this.props;
return (
<>
{input.length <= 1 && (
<div>
<FieldValidationMessage>Filter data by query expects multiple queries in the input.</FieldValidationMessage>
</div>
)}
<div className="gf-form-inline">
<div className="gf-form gf-form--grow">
<div className="gf-form-label width-8">Series refId</div>
@ -125,6 +132,7 @@ export class FilterByRefIdTransformerEditor extends React.PureComponent<
</HorizontalGroup>
</div>
</div>
</>
);
}
}

6
public/app/features/transformers/editors/OrganizeFieldsTransformerEditor.tsx
View File

@ -17,8 +17,7 @@ import { useAllFieldNamesFromDataFrames } from '../utils';
interface OrganizeFieldsTransformerEditorProps extends TransformerUIProps<OrganizeFieldsTransformerOptions> {}
const OrganizeFieldsTransformerEditor: React.FC<OrganizeFieldsTransformerEditorProps> = (props) => {
const { options, input, onChange } = props;
const OrganizeFieldsTransformerEditor = ({ options, input, onChange }: OrganizeFieldsTransformerEditorProps) => {
const { indexByName, excludeByName, renameByName } = options;
const fieldNames = useAllFieldNamesFromDataFrames(input);
@ -75,7 +74,8 @@ const OrganizeFieldsTransformerEditor: React.FC<OrganizeFieldsTransformerEditorP
if (input.length > 1) {
return (
<FieldValidationMessage>
Organize fields only works with a single frame. Consider applying a join transformation first.
Organize fields only works with a single frame. Consider applying a join transformation or filtering the input
first.
</FieldValidationMessage>
);
}