diff --git a/packages/grafana-data/src/utils/index.ts b/packages/grafana-data/src/utils/index.ts index fa926939558..2479adafe58 100644 --- a/packages/grafana-data/src/utils/index.ts +++ b/packages/grafana-data/src/utils/index.ts @@ -22,3 +22,8 @@ export { getMappedValue } from './valueMappings'; import * as dateMath from './datemath'; import * as rangeUtil from './rangeutil'; export { dateMath, rangeUtil }; + +export * from './matchers/ids'; +export * from './matchers/matchers'; +export * from './transformers/ids'; +export * from './transformers/transformers'; diff --git a/packages/grafana-data/src/utils/matchers/fieldTypeMatcher.test.ts b/packages/grafana-data/src/utils/matchers/fieldTypeMatcher.test.ts new file mode 100644 index 00000000000..99884f3d60c --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/fieldTypeMatcher.test.ts @@ -0,0 +1,22 @@ +import { FieldType } from '../../types/dataFrame'; +import { fieldMatchers } from './matchers'; +import { FieldMatcherID } from './ids'; +import { toDataFrame } from '../processDataFrame'; + +export const simpleSeriesWithTypes = toDataFrame({ + fields: [ + { name: 'A', type: FieldType.time }, + { name: 'B', type: FieldType.boolean }, + { name: 'C', type: FieldType.string }, + ], +}); + +describe('Field Type Matcher', () => { + const matcher = fieldMatchers.get(FieldMatcherID.byType); + it('finds numbers', () => { + for (const field of simpleSeriesWithTypes.fields) { + const matches = matcher.get(FieldType.number); + expect(matches(field)).toBe(field.type === FieldType.number); + } + }); +}); diff --git a/packages/grafana-data/src/utils/matchers/fieldTypeMatcher.ts b/packages/grafana-data/src/utils/matchers/fieldTypeMatcher.ts new file mode 100644 index 00000000000..385bcefca44 --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/fieldTypeMatcher.ts @@ -0,0 +1,59 @@ +import { Field, FieldType } from '../../types/dataFrame'; +import { FieldMatcherInfo } from './matchers'; +import { FieldMatcherID } from './ids'; + +// General Field matcher +const fieldTypeMacher: FieldMatcherInfo = { + id: FieldMatcherID.byType, + name: 'Field Type', + description: 'match based on the field type', + defaultOptions: FieldType.number, + + get: (type: FieldType) => { + return (field: Field) => { + return type === field.type; + }; + }, + + getOptionsDisplayText: (type: FieldType) => { + return `Field type: ${type}`; + }, +}; + +// Numeric Field matcher +// This gets its own entry so it shows up in the dropdown +const numericMacher: FieldMatcherInfo = { + id: FieldMatcherID.numeric, + name: 'Numeric Fields', + description: 'Fields with type number', + + get: () => { + return fieldTypeMacher.get(FieldType.number); + }, + + getOptionsDisplayText: () => { + return 'Numeric Fields'; + }, +}; + +// Time Field matcher +const timeMacher: FieldMatcherInfo = { + id: FieldMatcherID.time, + name: 'Time Fields', + description: 'Fields with type time', + + get: () => { + return fieldTypeMacher.get(FieldType.time); + }, + + getOptionsDisplayText: () => { + return 'Time Fields'; + }, +}; + +/** + * Registry Initalization + */ +export function getFieldTypeMatchers(): FieldMatcherInfo[] { + return [fieldTypeMacher, numericMacher, timeMacher]; +} diff --git a/packages/grafana-data/src/utils/matchers/ids.ts b/packages/grafana-data/src/utils/matchers/ids.ts new file mode 100644 index 00000000000..ee57678e26f --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/ids.ts @@ -0,0 +1,33 @@ +// This needs to be in its own file to avoid circular references + +// Builtin Predicates +// not using 'any' and 'never' since they are reservered keywords +export enum MatcherID { + anyMatch = 'anyMatch', // checks children + allMatch = 'allMatch', // checks children + invertMatch = 'invertMatch', // checks child + alwaysMatch = 'alwaysMatch', + neverMatch = 'neverMatch', +} + +export enum FieldMatcherID { + // Specific Types + numeric = 'numeric', + time = 'time', + + // With arguments + byType = 'byType', + byName = 'byName', + // byIndex = 'byIndex', + // byLabel = 'byLabel', +} + +/** + * Field name matchers + */ +export enum FrameMatcherID { + byName = 'byName', + byRefId = 'byRefId', + byIndex = 'byIndex', + byLabel = 'byLabel', +} diff --git a/packages/grafana-data/src/utils/matchers/matchers.test.ts b/packages/grafana-data/src/utils/matchers/matchers.test.ts new file mode 100644 index 00000000000..0faeabb14ae --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/matchers.test.ts @@ -0,0 +1,11 @@ +import { fieldMatchers } from './matchers'; +import { FieldMatcherID } from './ids'; + +describe('Matchers', () => { + it('should load all matchers', () => { + for (const name of Object.keys(FieldMatcherID)) { + const matcher = fieldMatchers.get(name); + expect(matcher.id).toBe(name); + } + }); +}); diff --git a/packages/grafana-data/src/utils/matchers/matchers.ts b/packages/grafana-data/src/utils/matchers/matchers.ts new file mode 100644 index 00000000000..d5b45ff771a --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/matchers.ts @@ -0,0 +1,56 @@ +import { Field, DataFrame } from '../../types/dataFrame'; +import { Registry, RegistryItemWithOptions } from '../registry'; + +export type FieldMatcher = (field: Field) => boolean; +export type FrameMatcher = (frame: DataFrame) => boolean; + +export interface FieldMatcherInfo extends RegistryItemWithOptions { + get: (options: TOptions) => FieldMatcher; +} + +export interface FrameMatcherInfo extends RegistryItemWithOptions { + get: (options: TOptions) => FrameMatcher; +} + +export interface MatcherConfig { + id: string; + options?: TOptions; +} + +// Load the Buildtin matchers +import { getFieldPredicateMatchers, getFramePredicateMatchers } from './predicates'; +import { getFieldNameMatchers, getFrameNameMatchers } from './nameMatcher'; +import { getFieldTypeMatchers } from './fieldTypeMatcher'; +import { getRefIdMatchers } from './refIdMatcher'; + +export const fieldMatchers = new Registry(() => { + return [ + ...getFieldPredicateMatchers(), // Predicates + ...getFieldTypeMatchers(), // by type + ...getFieldNameMatchers(), // by name + ]; +}); + +export const frameMatchers = new Registry(() => { + return [ + ...getFramePredicateMatchers(), // Predicates + ...getFrameNameMatchers(), // by name + ...getRefIdMatchers(), // by query refId + ]; +}); + +export function getFieldMatcher(config: MatcherConfig): FieldMatcher { + const info = fieldMatchers.get(config.id); + if (!info) { + throw new Error('Unknown Matcher: ' + config.id); + } + return info.get(config.options); +} + +export function getFrameMatchers(config: MatcherConfig): FrameMatcher { + const info = frameMatchers.get(config.id); + if (!info) { + throw new Error('Unknown Matcher: ' + config.id); + } + return info.get(config.options); +} diff --git a/packages/grafana-data/src/utils/matchers/nameMatcher.test.ts b/packages/grafana-data/src/utils/matchers/nameMatcher.test.ts new file mode 100644 index 00000000000..7f2880ff0f6 --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/nameMatcher.test.ts @@ -0,0 +1,56 @@ +import { getFieldMatcher } from './matchers'; +import { FieldMatcherID } from './ids'; +import { toDataFrame } from '../processDataFrame'; + +describe('Field Name Matcher', () => { + it('Match all with wildcard regex', () => { + const seriesWithNames = toDataFrame({ + fields: [{ name: 'A hello world' }, { name: 'AAA' }, { name: 'C' }], + }); + const config = { + id: FieldMatcherID.byName, + options: '/.*/', + }; + + const matcher = getFieldMatcher(config); + + for (const field of seriesWithNames.fields) { + expect(matcher(field)).toBe(true); + } + }); + + it('Match all with decimals regex', () => { + const seriesWithNames = toDataFrame({ + fields: [{ name: '12' }, { name: '112' }, { name: '13' }], + }); + const config = { + id: FieldMatcherID.byName, + options: '/^\\d+$/', + }; + + const matcher = getFieldMatcher(config); + + for (const field of seriesWithNames.fields) { + expect(matcher(field)).toBe(true); + } + }); + + it('Match complex regex', () => { + const seriesWithNames = toDataFrame({ + fields: [{ name: 'some.instance.path' }, { name: '112' }, { name: '13' }], + }); + const config = { + id: FieldMatcherID.byName, + options: '/\\b(?:\\S+?\\.)+\\S+\\b$/', + }; + + const matcher = getFieldMatcher(config); + let resultCount = 0; + for (const field of seriesWithNames.fields) { + if (matcher(field)) { + resultCount++; + } + expect(resultCount).toBe(1); + } + }); +}); diff --git a/packages/grafana-data/src/utils/matchers/nameMatcher.ts b/packages/grafana-data/src/utils/matchers/nameMatcher.ts new file mode 100644 index 00000000000..626b6a908a8 --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/nameMatcher.ts @@ -0,0 +1,53 @@ +import { Field, DataFrame } from '../../types/dataFrame'; +import { FieldMatcherInfo, FrameMatcherInfo } from './matchers'; +import { FieldMatcherID, FrameMatcherID } from './ids'; +import { stringToJsRegex } from '../string'; + +// General Field matcher +const fieldNameMacher: FieldMatcherInfo = { + id: FieldMatcherID.byName, + name: 'Field Name', + description: 'match the field name', + defaultOptions: '/.*/', + + get: (pattern: string) => { + const regex = stringToJsRegex(pattern); + return (field: Field) => { + return regex.test(field.name); + }; + }, + + getOptionsDisplayText: (pattern: string) => { + return `Field name: ${pattern}`; + }, +}; + +// General Field matcher +const frameNameMacher: FrameMatcherInfo = { + id: FrameMatcherID.byName, + name: 'Frame Name', + description: 'match the frame name', + defaultOptions: '/.*/', + + get: (pattern: string) => { + const regex = stringToJsRegex(pattern); + return (frame: DataFrame) => { + return regex.test(frame.name || ''); + }; + }, + + getOptionsDisplayText: (pattern: string) => { + return `Frame name: ${pattern}`; + }, +}; + +/** + * Registry Initalization + */ +export function getFieldNameMatchers(): FieldMatcherInfo[] { + return [fieldNameMacher]; +} + +export function getFrameNameMatchers(): FrameMatcherInfo[] { + return [frameNameMacher]; +} diff --git a/packages/grafana-data/src/utils/matchers/predicates.test.ts b/packages/grafana-data/src/utils/matchers/predicates.test.ts new file mode 100644 index 00000000000..97e95ba3129 --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/predicates.test.ts @@ -0,0 +1,37 @@ +import { FieldType } from '../../types/dataFrame'; +import { MatcherConfig, fieldMatchers } from './matchers'; +import { simpleSeriesWithTypes } from './fieldTypeMatcher.test'; +import { FieldMatcherID, MatcherID } from './ids'; + +const matchesNumberConfig: MatcherConfig = { + id: FieldMatcherID.byType, + options: FieldType.number, +}; +const matchesTimeConfig: MatcherConfig = { + id: FieldMatcherID.byType, + options: FieldType.time, +}; +const both = [matchesNumberConfig, matchesTimeConfig]; + +describe('Check Predicates', () => { + it('can not match both', () => { + const matches = fieldMatchers.get(MatcherID.allMatch).get(both); + for (const field of simpleSeriesWithTypes.fields) { + expect(matches(field)).toBe(false); + } + }); + + it('match either time or number', () => { + const matches = fieldMatchers.get(MatcherID.anyMatch).get(both); + for (const field of simpleSeriesWithTypes.fields) { + expect(matches(field)).toBe(field.type === FieldType.number || field.type === FieldType.time); + } + }); + + it('match not time', () => { + const matches = fieldMatchers.get(MatcherID.invertMatch).get(matchesTimeConfig); + for (const field of simpleSeriesWithTypes.fields) { + expect(matches(field)).toBe(field.type !== FieldType.time); + } + }); +}); diff --git a/packages/grafana-data/src/utils/matchers/predicates.ts b/packages/grafana-data/src/utils/matchers/predicates.ts new file mode 100644 index 00000000000..502cceef311 --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/predicates.ts @@ -0,0 +1,268 @@ +import { Field, DataFrame } from '../../types/dataFrame'; +import { MatcherID } from './ids'; +import { + FrameMatcherInfo, + FieldMatcherInfo, + MatcherConfig, + getFieldMatcher, + fieldMatchers, + getFrameMatchers, + frameMatchers, +} from './matchers'; + +const anyFieldMatcher: FieldMatcherInfo = { + id: MatcherID.anyMatch, + name: 'Any', + description: 'Any child matches (OR)', + excludeFromPicker: true, + defaultOptions: [], // empty array + + get: (options: MatcherConfig[]) => { + const children = options.map(option => { + return getFieldMatcher(option); + }); + return (field: Field) => { + for (const child of children) { + if (child(field)) { + return true; + } + } + return false; + }; + }, + + getOptionsDisplayText: (options: MatcherConfig[]) => { + let text = ''; + for (const sub of options) { + if (text.length > 0) { + text += ' OR '; + } + const matcher = fieldMatchers.get(sub.id); + text += matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(sub) : matcher.name; + } + return text; + }, +}; + +const anyFrameMatcher: FrameMatcherInfo = { + id: MatcherID.anyMatch, + name: 'Any', + description: 'Any child matches (OR)', + excludeFromPicker: true, + defaultOptions: [], // empty array + + get: (options: MatcherConfig[]) => { + const children = options.map(option => { + return getFrameMatchers(option); + }); + return (frame: DataFrame) => { + for (const child of children) { + if (child(frame)) { + return true; + } + } + return false; + }; + }, + + getOptionsDisplayText: (options: MatcherConfig[]) => { + let text = ''; + for (const sub of options) { + if (text.length > 0) { + text += ' OR '; + } + const matcher = frameMatchers.get(sub.id); + text += matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(sub) : matcher.name; + } + return text; + }, +}; + +const allFieldsMatcher: FieldMatcherInfo = { + id: MatcherID.allMatch, + name: 'All', + description: 'Everything matches (AND)', + excludeFromPicker: true, + defaultOptions: [], // empty array + + get: (options: MatcherConfig[]) => { + const children = options.map(option => { + return getFieldMatcher(option); + }); + return (field: Field) => { + for (const child of children) { + if (!child(field)) { + return false; + } + } + return true; + }; + }, + + getOptionsDisplayText: (options: MatcherConfig[]) => { + let text = ''; + for (const sub of options) { + if (text.length > 0) { + text += ' AND '; + } + const matcher = fieldMatchers.get(sub.id); // Ugho what about frame + text += matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(sub) : matcher.name; + } + return text; + }, +}; + +const allFramesMatcher: FrameMatcherInfo = { + id: MatcherID.allMatch, + name: 'All', + description: 'Everything matches (AND)', + excludeFromPicker: true, + defaultOptions: [], // empty array + + get: (options: MatcherConfig[]) => { + const children = options.map(option => { + return getFrameMatchers(option); + }); + return (frame: DataFrame) => { + for (const child of children) { + if (!child(frame)) { + return false; + } + } + return true; + }; + }, + + getOptionsDisplayText: (options: MatcherConfig[]) => { + let text = ''; + for (const sub of options) { + if (text.length > 0) { + text += ' AND '; + } + const matcher = frameMatchers.get(sub.id); + text += matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(sub) : matcher.name; + } + return text; + }, +}; + +const notFieldMatcher: FieldMatcherInfo = { + id: MatcherID.invertMatch, + name: 'NOT', + description: 'Inverts other matchers', + excludeFromPicker: true, + + get: (option: MatcherConfig) => { + const check = getFieldMatcher(option); + return (field: Field) => { + return !check(field); + }; + }, + + getOptionsDisplayText: (options: MatcherConfig) => { + const matcher = fieldMatchers.get(options.id); + const text = matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(options.options) : matcher.name; + return 'NOT ' + text; + }, +}; + +const notFrameMatcher: FrameMatcherInfo = { + id: MatcherID.invertMatch, + name: 'NOT', + description: 'Inverts other matchers', + excludeFromPicker: true, + + get: (option: MatcherConfig) => { + const check = getFrameMatchers(option); + return (frame: DataFrame) => { + return !check(frame); + }; + }, + + getOptionsDisplayText: (options: MatcherConfig) => { + const matcher = frameMatchers.get(options.id); + const text = matcher.getOptionsDisplayText ? matcher.getOptionsDisplayText(options.options) : matcher.name; + return 'NOT ' + text; + }, +}; + +export const alwaysFieldMatcher = (field: Field) => { + return true; +}; + +export const alwaysFrameMatcher = (frame: DataFrame) => { + return true; +}; + +export const neverFieldMatcher = (field: Field) => { + return false; +}; + +export const neverFrameMatcher = (frame: DataFrame) => { + return false; +}; + +const alwaysFieldMatcherInfo: FieldMatcherInfo = { + id: MatcherID.alwaysMatch, + name: 'All Fields', + description: 'Always Match', + + get: (option: any) => { + return alwaysFieldMatcher; + }, + + getOptionsDisplayText: (options: any) => { + return 'Always'; + }, +}; + +const alwaysFrameMatcherInfo: FrameMatcherInfo = { + id: MatcherID.alwaysMatch, + name: 'All Frames', + description: 'Always Match', + + get: (option: any) => { + return alwaysFrameMatcher; + }, + + getOptionsDisplayText: (options: any) => { + return 'Always'; + }, +}; + +const neverFieldMatcherInfo: FieldMatcherInfo = { + id: MatcherID.neverMatch, + name: 'No Fields', + description: 'Never Match', + excludeFromPicker: true, + + get: (option: any) => { + return neverFieldMatcher; + }, + + getOptionsDisplayText: (options: any) => { + return 'Never'; + }, +}; + +const neverFrameMatcherInfo: FrameMatcherInfo = { + id: MatcherID.neverMatch, + name: 'No Frames', + description: 'Never Match', + + get: (option: any) => { + return neverFrameMatcher; + }, + + getOptionsDisplayText: (options: any) => { + return 'Never'; + }, +}; + +export function getFieldPredicateMatchers(): FieldMatcherInfo[] { + return [anyFieldMatcher, allFieldsMatcher, notFieldMatcher, alwaysFieldMatcherInfo, neverFieldMatcherInfo]; +} + +export function getFramePredicateMatchers(): FrameMatcherInfo[] { + return [anyFrameMatcher, allFramesMatcher, notFrameMatcher, alwaysFrameMatcherInfo, neverFrameMatcherInfo]; +} diff --git a/packages/grafana-data/src/utils/matchers/refIdMatcher.ts b/packages/grafana-data/src/utils/matchers/refIdMatcher.ts new file mode 100644 index 00000000000..51b0db3af80 --- /dev/null +++ b/packages/grafana-data/src/utils/matchers/refIdMatcher.ts @@ -0,0 +1,25 @@ +import { DataFrame } from '../../types/dataFrame'; +import { FrameMatcherInfo } from './matchers'; +import { FrameMatcherID } from './ids'; + +// General Field matcher +const refIdMacher: FrameMatcherInfo = { + id: FrameMatcherID.byRefId, + name: 'Query refId', + description: 'match the refId', + defaultOptions: 'A', + + get: (pattern: string) => { + return (frame: DataFrame) => { + return pattern === frame.refId; + }; + }, + + getOptionsDisplayText: (pattern: string) => { + return `RefID: ${pattern}`; + }, +}; + +export function getRefIdMatchers(): FrameMatcherInfo[] { + return [refIdMacher]; +} diff --git a/packages/grafana-data/src/utils/registry.ts b/packages/grafana-data/src/utils/registry.ts index bc4ce20091b..ffe8f498500 100644 --- a/packages/grafana-data/src/utils/registry.ts +++ b/packages/grafana-data/src/utils/registry.ts @@ -13,6 +13,18 @@ export interface RegistryItem { excludeFromPicker?: boolean; } +export interface RegistryItemWithOptions extends RegistryItem { + /** + * Convert the options to a string + */ + getOptionsDisplayText?: (options: TOptions) => string; + + /** + * Default options used if nothing else is specified + */ + defaultOptions?: TOptions; +} + interface RegistrySelectInfo { options: Array>; current: Array>; diff --git a/packages/grafana-data/src/utils/transformers/__snapshots__/reduce.test.ts.snap b/packages/grafana-data/src/utils/transformers/__snapshots__/reduce.test.ts.snap new file mode 100644 index 00000000000..fab0404045e --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/__snapshots__/reduce.test.ts.snap @@ -0,0 +1,69 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`Reducer Transformer filters by include 1`] = ` +Object { + "fields": Array [ + Object { + "config": Object {}, + "name": "Field", + "type": "string", + "values": Array [ + "A", + "B", + ], + }, + Object { + "config": Object { + "title": "First", + }, + "name": "first", + "type": "number", + "values": Array [ + 1, + "a", + ], + }, + Object { + "config": Object { + "title": "Min", + }, + "name": "min", + "type": "number", + "values": Array [ + 1, + null, + ], + }, + Object { + "config": Object { + "title": "Max", + }, + "name": "max", + "type": "number", + "values": Array [ + 4, + null, + ], + }, + Object { + "config": Object { + "title": "Delta", + }, + "name": "delta", + "type": "number", + "values": Array [ + 3, + 0, + ], + }, + ], + "labels": undefined, + "meta": Object { + "transformations": Array [ + "reduce", + ], + }, + "name": undefined, + "refId": undefined, +} +`; diff --git a/packages/grafana-data/src/utils/transformers/append.test.ts b/packages/grafana-data/src/utils/transformers/append.test.ts new file mode 100644 index 00000000000..5e9c8feaefd --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/append.test.ts @@ -0,0 +1,41 @@ +import { transformDataFrame, dataTransformers } from './transformers'; +import { DataTransformerID } from './ids'; +import { toDataFrame } from '../processDataFrame'; + +const seriesAB = toDataFrame({ + columns: [{ text: 'A' }, { text: 'B' }], + rows: [ + [1, 100], // A,B + [2, 200], // A,B + ], +}); + +const seriesBC = toDataFrame({ + columns: [{ text: 'A' }, { text: 'C' }], + rows: [ + [3, 3000], // A,C + [4, 4000], // A,C + ], +}); + +describe('Append Transformer', () => { + it('filters by include', () => { + const cfg = { + id: DataTransformerID.append, + options: {}, + }; + const x = dataTransformers.get(DataTransformerID.append); + expect(x.id).toBe(cfg.id); + + const processed = transformDataFrame([cfg], [seriesAB, seriesBC])[0]; + expect(processed.fields.length).toBe(3); + + const fieldA = processed.fields[0]; + const fieldB = processed.fields[1]; + const fieldC = processed.fields[2]; + + expect(fieldA.values.toArray()).toEqual([1, 2, 3, 4]); + expect(fieldB.values.toArray()).toEqual([100, 200, undefined, undefined]); + expect(fieldC.values.toArray()).toEqual([undefined, undefined, 3000, 4000]); + }); +}); diff --git a/packages/grafana-data/src/utils/transformers/append.ts b/packages/grafana-data/src/utils/transformers/append.ts new file mode 100644 index 00000000000..9129033c735 --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/append.ts @@ -0,0 +1,58 @@ +import { DataTransformerInfo } from './transformers'; +import { DataFrame } from '../../types/dataFrame'; +import { DataTransformerID } from './ids'; +import { DataFrameHelper } from '../dataFrameHelper'; +import { KeyValue } from '../../types/data'; +import { AppendedVectors } from '../vector'; + +export interface AppendOptions {} + +export const appendTransformer: DataTransformerInfo = { + id: DataTransformerID.append, + name: 'Append', + description: 'Append values into a single DataFrame. This uses the name as the key', + defaultOptions: {}, + + /** + * Return a modified copy of the series. If the transform is not or should not + * be applied, just return the input series + */ + transformer: (options: AppendOptions) => { + return (data: DataFrame[]) => { + if (data.length < 2) { + return data; + } + + let length = 0; + const processed = new DataFrameHelper(); + for (let i = 0; i < data.length; i++) { + const frame = data[i]; + const used: KeyValue = {}; + for (let j = 0; j < frame.fields.length; j++) { + const src = frame.fields[j]; + if (used[src.name]) { + continue; + } + used[src.name] = true; + + let f = processed.getFieldByName(src.name); + if (!f) { + f = processed.addField({ + ...src, + values: new AppendedVectors(length), + }); + } + (f.values as AppendedVectors).append(src.values); + } + + // Make sure all fields have their length updated + length += frame.length; + processed.length = length; + for (const f of processed.fields) { + (f.values as AppendedVectors).setLength(processed.length); + } + } + return [processed]; + }; + }, +}; diff --git a/packages/grafana-data/src/utils/transformers/filter.test.ts b/packages/grafana-data/src/utils/transformers/filter.test.ts new file mode 100644 index 00000000000..28b3c11f6f7 --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/filter.test.ts @@ -0,0 +1,29 @@ +import { FieldType } from '../../types/dataFrame'; +import { FieldMatcherID } from '../matchers/ids'; +import { transformDataFrame } from './transformers'; +import { DataTransformerID } from './ids'; +import { toDataFrame } from '../processDataFrame'; + +export const simpleSeriesWithTypes = toDataFrame({ + fields: [ + { name: 'A', type: FieldType.time, values: [1000, 2000] }, + { name: 'B', type: FieldType.boolean, values: [true, false] }, + { name: 'C', type: FieldType.string, values: ['a', 'b'] }, + { name: 'D', type: FieldType.number, values: [1, 2] }, + ], +}); + +describe('Filter Transformer', () => { + it('filters by include', () => { + const cfg = { + id: DataTransformerID.filterFields, + options: { + include: { id: FieldMatcherID.numeric }, + }, + }; + + const filtered = transformDataFrame([cfg], [simpleSeriesWithTypes])[0]; + expect(filtered.fields.length).toBe(1); + expect(filtered.fields[0].name).toBe('D'); + }); +}); diff --git a/packages/grafana-data/src/utils/transformers/filter.ts b/packages/grafana-data/src/utils/transformers/filter.ts new file mode 100644 index 00000000000..02083e3d52b --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/filter.ts @@ -0,0 +1,102 @@ +import { DataTransformerInfo, NoopDataTransformer } from './transformers'; +import { DataFrame, Field } from '../../types/dataFrame'; +import { FieldMatcherID } from '../matchers/ids'; +import { DataTransformerID } from './ids'; +import { MatcherConfig, getFieldMatcher, getFrameMatchers } from '../matchers/matchers'; + +export interface FilterOptions { + include?: MatcherConfig; + exclude?: MatcherConfig; +} + +export const filterFieldsTransformer: DataTransformerInfo = { + id: DataTransformerID.filterFields, + name: 'Filter Fields', + description: 'select a subset of fields', + defaultOptions: { + include: { id: FieldMatcherID.numeric }, + }, + + /** + * Return a modified copy of the series. If the transform is not or should not + * be applied, just return the input series + */ + transformer: (options: FilterOptions) => { + if (!options.include && !options.exclude) { + return NoopDataTransformer; + } + + const include = options.include ? getFieldMatcher(options.include) : null; + const exclude = options.exclude ? getFieldMatcher(options.exclude) : null; + + return (data: DataFrame[]) => { + const processed: DataFrame[] = []; + for (const series of data) { + // Find the matching field indexes + const fields: Field[] = []; + for (let i = 0; i < series.fields.length; i++) { + const field = series.fields[i]; + if (exclude) { + if (exclude(field)) { + continue; + } + if (!include) { + fields.push(field); + } + } + if (include && include(field)) { + fields.push(field); + } + } + + if (!fields.length) { + continue; + } + const copy = { + ...series, // all the other properties + fields, // but a different set of fields + }; + processed.push(copy); + } + return processed; + }; + }, +}; + +export const filterFramesTransformer: DataTransformerInfo = { + id: DataTransformerID.filterFrames, + name: 'Filter Frames', + description: 'select a subset of frames', + defaultOptions: {}, + + /** + * Return a modified copy of the series. If the transform is not or should not + * be applied, just return the input series + */ + transformer: (options: FilterOptions) => { + if (!options.include && !options.exclude) { + return NoopDataTransformer; + } + + const include = options.include ? getFrameMatchers(options.include) : null; + const exclude = options.exclude ? getFrameMatchers(options.exclude) : null; + + return (data: DataFrame[]) => { + const processed: DataFrame[] = []; + for (const series of data) { + if (exclude) { + if (exclude(series)) { + continue; + } + if (!include) { + processed.push(series); + } + } + if (include && include(series)) { + processed.push(series); + } + } + return processed; + }; + }, +}; diff --git a/packages/grafana-data/src/utils/transformers/ids.ts b/packages/grafana-data/src/utils/transformers/ids.ts new file mode 100644 index 00000000000..32fab351371 --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/ids.ts @@ -0,0 +1,9 @@ +export enum DataTransformerID { + // join = 'join', // Pick a field and merge all series based on that field + append = 'append', // Merge all series together + // rotate = 'rotate', // Columns to rows + reduce = 'reduce', // Run calculations on fields + + filterFields = 'filterFields', // Pick some fields (keep all frames) + filterFrames = 'filterFrames', // Pick some frames (keep all fields) +} diff --git a/packages/grafana-data/src/utils/transformers/reduce.test.ts b/packages/grafana-data/src/utils/transformers/reduce.test.ts new file mode 100644 index 00000000000..ed0ce16b6f7 --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/reduce.test.ts @@ -0,0 +1,25 @@ +import { transformDataFrame } from './transformers'; +import { ReducerID } from '../fieldReducer'; +import { DataTransformerID } from './ids'; +import { toDataFrame, toDataFrameDTO } from '../processDataFrame'; + +const seriesWithValues = toDataFrame({ + fields: [ + { name: 'A', values: [1, 2, 3, 4] }, // Numbers + { name: 'B', values: ['a', 'b', 'c', 'd'] }, // Strings + ], +}); + +describe('Reducer Transformer', () => { + it('filters by include', () => { + const cfg = { + id: DataTransformerID.reduce, + options: { + reducers: [ReducerID.first, ReducerID.min, ReducerID.max, ReducerID.delta], + }, + }; + const processed = transformDataFrame([cfg], [seriesWithValues])[0]; + expect(processed.fields.length).toBe(5); + expect(toDataFrameDTO(processed)).toMatchSnapshot(); + }); +}); diff --git a/packages/grafana-data/src/utils/transformers/reduce.ts b/packages/grafana-data/src/utils/transformers/reduce.ts new file mode 100644 index 00000000000..a70ecd13bbf --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/reduce.ts @@ -0,0 +1,90 @@ +import { DataTransformerInfo } from './transformers'; +import { DataFrame, FieldType, Field } from '../../types/dataFrame'; +import { MatcherConfig, getFieldMatcher } from '../matchers/matchers'; +import { alwaysFieldMatcher } from '../matchers/predicates'; +import { DataTransformerID } from './ids'; +import { ReducerID, fieldReducers, reduceField } from '../fieldReducer'; +import { KeyValue } from '../../types/data'; +import { ArrayVector } from '../vector'; +import { guessFieldTypeForField } from '../processDataFrame'; + +export interface ReduceOptions { + reducers: string[]; + fields?: MatcherConfig; // Assume all fields +} + +export const reduceTransformer: DataTransformerInfo = { + id: DataTransformerID.reduce, + name: 'Reducer', + description: 'Return a DataFrame with the reduction results', + defaultOptions: { + calcs: [ReducerID.min, ReducerID.max, ReducerID.mean, ReducerID.last], + }, + + /** + * Return a modified copy of the series. If the transform is not or should not + * be applied, just return the input series + */ + transformer: (options: ReduceOptions) => { + const matcher = options.fields ? getFieldMatcher(options.fields) : alwaysFieldMatcher; + const calculators = fieldReducers.list(options.reducers); + const reducers = calculators.map(c => c.id); + + return (data: DataFrame[]) => { + const processed: DataFrame[] = []; + for (const series of data) { + const values: ArrayVector[] = []; + const fields: Field[] = []; + const byId: KeyValue = {}; + values.push(new ArrayVector()); // The name + fields.push({ + name: 'Field', + type: FieldType.string, + values: values[0], + config: {}, + }); + for (const info of calculators) { + const vals = new ArrayVector(); + byId[info.id] = vals; + values.push(vals); + fields.push({ + name: info.id, + type: FieldType.other, // UNKNOWN until after we call the functions + values: values[values.length - 1], + config: { + title: info.name, + // UNIT from original field? + }, + }); + } + for (let i = 0; i < series.fields.length; i++) { + const field = series.fields[i]; + if (matcher(field)) { + const results = reduceField({ + field, + reducers, + }); + // Update the name list + values[0].buffer.push(field.name); + for (const info of calculators) { + const v = results[info.id]; + byId[info.id].buffer.push(v); + } + } + } + for (const f of fields) { + const t = guessFieldTypeForField(f); + if (t) { + f.type = t; + } + } + processed.push({ + ...series, // Same properties, different fields + fields, + length: values[0].length, + }); + } + return processed; + }; + }, +}; diff --git a/packages/grafana-data/src/utils/transformers/transformers.test.ts b/packages/grafana-data/src/utils/transformers/transformers.test.ts new file mode 100644 index 00000000000..81df7478d59 --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/transformers.test.ts @@ -0,0 +1,34 @@ +import { DataTransformerID } from './ids'; +import { dataTransformers } from './transformers'; +import { toDataFrame } from '../processDataFrame'; +import { ReducerID } from '../fieldReducer'; +import { DataFrameView } from '../dataFrameView'; + +describe('Transformers', () => { + it('should load all transformeres', () => { + for (const name of Object.keys(DataTransformerID)) { + const calc = dataTransformers.get(name); + expect(calc.id).toBe(name); + } + }); + + const seriesWithValues = toDataFrame({ + fields: [ + { name: 'A', values: [1, 2, 3, 4] }, // Numbers + { name: 'B', values: ['a', 'b', 'c', 'd'] }, // Strings + ], + }); + + it('should use fluent API', () => { + const results = dataTransformers.reduce([seriesWithValues], { + reducers: [ReducerID.first], + }); + expect(results.length).toBe(1); + + const view = new DataFrameView(results[0]).toJSON(); + expect(view).toEqual([ + { Field: 'A', first: 1 }, // Row 0 + { Field: 'B', first: 'a' }, // Row 1 + ]); + }); +}); diff --git a/packages/grafana-data/src/utils/transformers/transformers.ts b/packages/grafana-data/src/utils/transformers/transformers.ts new file mode 100644 index 00000000000..9776ace625a --- /dev/null +++ b/packages/grafana-data/src/utils/transformers/transformers.ts @@ -0,0 +1,82 @@ +import { DataFrame } from '../../types/dataFrame'; +import { Registry, RegistryItemWithOptions } from '../registry'; + +/** + * Immutable data transformation + */ +export type DataTransformer = (data: DataFrame[]) => DataFrame[]; + +export interface DataTransformerInfo extends RegistryItemWithOptions { + transformer: (options: TOptions) => DataTransformer; +} + +export interface DataTransformerConfig { + id: string; + options: TOptions; +} + +// Transformer that does nothing +export const NoopDataTransformer = (data: DataFrame[]) => data; + +/** + * Apply configured transformations to the input data + */ +export function transformDataFrame(options: DataTransformerConfig[], data: DataFrame[]): DataFrame[] { + let processed = data; + for (const config of options) { + const info = dataTransformers.get(config.id); + const transformer = info.transformer(config.options); + const after = transformer(processed); + + // Add a key to the metadata if the data changed + if (after && after !== processed) { + for (const series of after) { + if (!series.meta) { + series.meta = {}; + } + if (!series.meta.transformations) { + series.meta.transformations = [info.id]; + } else { + series.meta.transformations = [...series.meta.transformations, info.id]; + } + } + processed = after; + } + } + return processed; +} + +// Initalize the Registry + +import { appendTransformer, AppendOptions } from './append'; +import { reduceTransformer, ReduceOptions } from './reduce'; +import { filterFieldsTransformer, filterFramesTransformer } from './filter'; + +/** + * Registry of transformation options that can be driven by + * stored configuration files. + */ +class TransformerRegistry extends Registry { + // ------------------------------------------------------------ + // Nacent options for more functional programming + // The API to these functions should change to match the actual + // needs of people trying to use it. + // filterFields|Frames is left off since it is likely easier to + // support with `frames.filter( f => {...} )` + // ------------------------------------------------------------ + + append(data: DataFrame[], options?: AppendOptions): DataFrame | undefined { + return appendTransformer.transformer(options || appendTransformer.defaultOptions)(data)[0]; + } + + reduce(data: DataFrame[], options: ReduceOptions): DataFrame[] { + return reduceTransformer.transformer(options)(data); + } +} + +export const dataTransformers = new TransformerRegistry(() => [ + filterFieldsTransformer, + filterFramesTransformer, + appendTransformer, + reduceTransformer, +]); diff --git a/packages/grafana-data/src/utils/vector.test.ts b/packages/grafana-data/src/utils/vector.test.ts index d99cf980c3e..1806020ed59 100644 --- a/packages/grafana-data/src/utils/vector.test.ts +++ b/packages/grafana-data/src/utils/vector.test.ts @@ -1,4 +1,4 @@ -import { ConstantVector, ScaledVector, ArrayVector, CircularVector } from './vector'; +import { ConstantVector, ScaledVector, ArrayVector, CircularVector, AppendedVectors } from './vector'; describe('Check Proxy Vector', () => { it('should support constant values', () => { @@ -156,3 +156,24 @@ describe('Check Circular Vector', () => { expect(v.toArray()).toEqual([3, 4, 5]); }); }); + +describe('Check Appending Vector', () => { + it('should transparently join them', () => { + const appended = new AppendedVectors(); + appended.append(new ArrayVector([1, 2, 3])); + appended.append(new ArrayVector([4, 5, 6])); + appended.append(new ArrayVector([7, 8, 9])); + expect(appended.length).toEqual(9); + + appended.setLength(5); + expect(appended.length).toEqual(5); + appended.append(new ArrayVector(['a', 'b', 'c'])); + expect(appended.length).toEqual(8); + expect(appended.toArray()).toEqual([1, 2, 3, 4, 5, 'a', 'b', 'c']); + + appended.setLength(2); + appended.setLength(6); + appended.append(new ArrayVector(['x', 'y', 'z'])); + expect(appended.toArray()).toEqual([1, 2, undefined, undefined, undefined, undefined, 'x', 'y', 'z']); + }); +}); diff --git a/packages/grafana-data/src/utils/vector.ts b/packages/grafana-data/src/utils/vector.ts index 0c8a34edbeb..339ebd9b06d 100644 --- a/packages/grafana-data/src/utils/vector.ts +++ b/packages/grafana-data/src/utils/vector.ts @@ -44,11 +44,8 @@ export class ConstantVector implements Vector { } toArray(): T[] { - const arr: T[] = []; - for (let i = 0; i < this.length; i++) { - arr[i] = this.value; - } - return arr; + const arr = new Array(this.length); + return arr.fill(this.value); } toJSON(): T[] { @@ -226,3 +223,74 @@ export class CircularVector implements Vector { return vectorToArray(this); } } + +interface AppendedVectorInfo { + start: number; + end: number; + values: Vector; +} + +/** + * This may be more trouble than it is worth. This trades some computation time for + * RAM -- rather than allocate a new array the size of all previous arrays, this just + * points the correct index to their original array values + */ +export class AppendedVectors implements Vector { + length = 0; + source: Array> = new Array>(); + + constructor(startAt = 0) { + this.length = startAt; + } + + /** + * Make the vector look like it is this long + */ + setLength(length: number) { + if (length > this.length) { + // make the vector longer (filling with undefined) + this.length = length; + } else if (length < this.length) { + // make the array shorter + const sources: Array> = new Array>(); + for (const src of this.source) { + sources.push(src); + if (src.end > length) { + src.end = length; + break; + } + } + this.source = sources; + this.length = length; + } + } + + append(v: Vector): AppendedVectorInfo { + const info = { + start: this.length, + end: this.length + v.length, + values: v, + }; + this.length = info.end; + this.source.push(info); + return info; + } + + get(index: number): T { + for (let i = 0; i < this.source.length; i++) { + const src = this.source[i]; + if (index >= src.start && index < src.end) { + return src.values.get(index - src.start); + } + } + return (undefined as unknown) as T; + } + + toArray(): T[] { + return vectorToArray(this); + } + + toJSON(): T[] { + return vectorToArray(this); + } +}