From a54a139176d287ffd24ec951d03be35ef2ed0282 Mon Sep 17 00:00:00 2001 From: nikki-kiga <42276368+nikki-kiga@users.noreply.github.com> Date: Fri, 27 Aug 2021 13:49:31 -0700 Subject: [PATCH] Transformations: Convert field types to time string number or boolean (#38517) * outline string to time add stringToTime transformer start to add format add type and dateformat rename stringToTime to fieldConversion add more type support and use FieldNamePicker add field conversion transformation * adjust for performance feedback rename and adjust labels and widths shorten labels and null values rename to convertFieldType update test * make updates --- .../src/field/overrides/processors.ts | 10 + .../grafana-data/src/transformations/index.ts | 1 + .../src/transformations/transformers.ts | 2 + .../transformers/convertFieldType.test.ts | 235 ++++++++++++++++++ .../transformers/convertFieldType.ts | 170 +++++++++++++ .../src/transformations/transformers/ids.ts | 1 + .../components/MatchersUI/FieldNamePicker.tsx | 2 + .../grafana-ui/src/components/uPlot/utils.ts | 13 +- .../ConvertFieldTypeTransformerEditor.tsx | 149 +++++++++++ public/app/core/utils/standardTransformers.ts | 2 + 10 files changed, 574 insertions(+), 11 deletions(-) create mode 100644 packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts create mode 100644 packages/grafana-data/src/transformations/transformers/convertFieldType.ts create mode 100644 public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx diff --git a/packages/grafana-data/src/field/overrides/processors.ts b/packages/grafana-data/src/field/overrides/processors.ts index a4aea5f46b5..84cb4d2a7fb 100644 --- a/packages/grafana-data/src/field/overrides/processors.ts +++ b/packages/grafana-data/src/field/overrides/processors.ts @@ -182,4 +182,14 @@ export interface FieldNamePickerConfigSettings { * information, including validation etc */ info?: ComponentType | null; + + /** + * Sets the width to a pixel value. + */ + width?: number; + + /** + * Placeholder text to display when nothing is selected. + */ + placeholderText?: string; } diff --git a/packages/grafana-data/src/transformations/index.ts b/packages/grafana-data/src/transformations/index.ts index 613f04e31bd..4a7104ddbd1 100644 --- a/packages/grafana-data/src/transformations/index.ts +++ b/packages/grafana-data/src/transformations/index.ts @@ -13,3 +13,4 @@ export { RegexpOrNamesMatcherOptions, ByNamesMatcherOptions, ByNamesMatcherMode export { RenameByRegexTransformerOptions } from './transformers/renameByRegex'; export { outerJoinDataFrames } from './transformers/joinDataFrames'; export * from './transformers/histogram'; +export { ensureTimeField } from './transformers/convertFieldType'; diff --git a/packages/grafana-data/src/transformations/transformers.ts b/packages/grafana-data/src/transformations/transformers.ts index deaed1a2e4c..835be0d0228 100644 --- a/packages/grafana-data/src/transformations/transformers.ts +++ b/packages/grafana-data/src/transformations/transformers.ts @@ -18,6 +18,7 @@ import { mergeTransformer } from './transformers/merge'; import { renameByRegexTransformer } from './transformers/renameByRegex'; import { filterByValueTransformer } from './transformers/filterByValue'; import { histogramTransformer } from './transformers/histogram'; +import { convertFieldTypeTransformer } from './transformers/convertFieldType'; export const standardTransformers = { noopTransformer, @@ -41,4 +42,5 @@ export const standardTransformers = { mergeTransformer, renameByRegexTransformer, histogramTransformer, + convertFieldTypeTransformer, }; diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts new file mode 100644 index 00000000000..cccbfb63b2f --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.test.ts @@ -0,0 +1,235 @@ +import { toDataFrame } from '../../dataframe/processDataFrame'; +import { FieldType } from '../../types/dataFrame'; +import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry'; +import { ArrayVector } from '../../vector'; +import { ensureTimeField, convertFieldType, convertFieldTypes, convertFieldTypeTransformer } from './convertFieldType'; + +describe('field convert type', () => { + it('will parse properly formatted strings to time', () => { + const options = { targetField: 'proper dates', destinationType: FieldType.time }; + + const stringTime = { + name: 'proper dates', + type: FieldType.string, + values: new ArrayVector([ + '2021-07-19 00:00:00.000', + '2021-07-23 00:00:00.000', + '2021-07-25 00:00:00.000', + '2021-08-01 00:00:00.000', + '2021-08-02 00:00:00.000', + ]), + config: {}, + }; + + const timefield = convertFieldType(stringTime, options); + expect(timefield).toEqual({ + name: 'proper dates', + type: FieldType.time, + values: new ArrayVector([1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000]), + config: {}, + }); + }); + it('will parse string time to specified format in time', () => { + const options = { targetField: 'format to year', destinationType: FieldType.time, dateFormat: 'YYYY' }; + + const yearFormat = { + name: 'format to year', + type: FieldType.string, + values: new ArrayVector([ + '2017-07-19 00:00:00.000', + '2018-07-23 00:00:00.000', + '2019-07-25 00:00:00.000', + '2020-08-01 00:00:00.000', + '2021-08-02 00:00:00.000', + ]), + config: {}, + }; + + const timefield = convertFieldType(yearFormat, options); + expect(timefield).toEqual({ + name: 'format to year', + type: FieldType.time, + values: new ArrayVector([1483246800000, 1514782800000, 1546318800000, 1577854800000, 1609477200000]), + config: {}, + }); + }); + + it('will not parse improperly formatted date strings', () => { + const options = { targetField: 'misformatted dates', destinationType: FieldType.time }; + + const misformattedStrings = { + name: 'misformatted dates', + type: FieldType.string, + values: new ArrayVector(['2021/08-01 00:00.00:000', '2021/08/01 00.00-000', '2021/08-01 00:00.00:000']), + config: { unit: 'time' }, + }; + + const timefield = convertFieldType(misformattedStrings, options); + expect(timefield).toEqual({ + name: 'misformatted dates', + type: FieldType.time, + values: new ArrayVector([null, null, null]), + config: { unit: 'time' }, + }); + }); + + it('can convert strings to numbers', () => { + const options = { targetField: 'stringy nums', destinationType: FieldType.number }; + + const stringyNumbers = { + name: 'stringy nums', + type: FieldType.string, + values: new ArrayVector(['10', '12', '30', '14', '10']), + config: {}, + }; + + const numbers = convertFieldType(stringyNumbers, options); + + expect(numbers).toEqual({ + name: 'stringy nums', + type: FieldType.number, + values: new ArrayVector([10, 12, 30, 14, 10]), + config: {}, + }); + }); +}); + +describe('field convert types transformer', () => { + beforeAll(() => { + mockTransformationsRegistry([convertFieldTypeTransformer]); + }); + it('can convert multiple fields', () => { + const options = { + conversions: [ + { targetField: 'stringy nums', destinationType: FieldType.number }, + { targetField: 'proper dates', destinationType: FieldType.time }, + ], + }; + + const stringyNumbers = toDataFrame({ + fields: [ + { name: 'A', type: FieldType.number, values: [1, 2, 3, 4, 5] }, + { + name: 'proper dates', + type: FieldType.string, + values: [ + '2021-07-19 00:00:00.000', + '2021-07-23 00:00:00.000', + '2021-07-25 00:00:00.000', + '2021-08-01 00:00:00.000', + '2021-08-02 00:00:00.000', + ], + }, + { name: 'stringy nums', type: FieldType.string, values: ['10', '12', '30', '14', '10'] }, + ], + }); + + const numbers = convertFieldTypes(options, [stringyNumbers]); + expect( + numbers[0].fields.map((f) => ({ + type: f.type, + values: f.values.toArray(), + })) + ).toEqual([ + { type: FieldType.number, values: [1, 2, 3, 4, 5] }, + { + type: FieldType.time, + values: [1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000], + }, + { + type: FieldType.number, + values: [10, 12, 30, 14, 10], + }, + ]); + }); + + it('will convert field to booleans', () => { + const options = { + conversions: [ + { targetField: 'numbers', destinationType: FieldType.boolean }, + { targetField: 'strings', destinationType: FieldType.boolean }, + ], + }; + + const comboTypes = toDataFrame({ + fields: [ + { name: 'numbers', type: FieldType.number, values: [-100, 0, 1, null, NaN] }, + { + name: 'strings', + type: FieldType.string, + values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'], + }, + ], + }); + + const booleans = convertFieldTypes(options, [comboTypes]); + expect( + booleans[0].fields.map((f) => ({ + type: f.type, + values: f.values.toArray(), + })) + ).toEqual([ + { + type: FieldType.boolean, + values: [true, false, true, false, false], + }, + { type: FieldType.boolean, values: [true, true, true, true, true] }, + ]); + }); + + it('will convert field to strings', () => { + const options = { + conversions: [{ targetField: 'numbers', destinationType: FieldType.string }], + }; + + const comboTypes = toDataFrame({ + fields: [ + { name: 'numbers', type: FieldType.number, values: [-100, 0, 1, null, NaN] }, + { + name: 'strings', + type: FieldType.string, + values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'], + }, + ], + }); + + const stringified = convertFieldTypes(options, [comboTypes]); + expect( + stringified[0].fields.map((f) => ({ + type: f.type, + values: f.values.toArray(), + })) + ).toEqual([ + { + type: FieldType.string, + values: ['-100', '0', '1', 'null', 'NaN'], + }, + { + type: FieldType.string, + values: ['true', 'false', '0', '99', '2021-08-02 00:00:00.000'], + }, + ]); + }); +}); + +describe('ensureTimeField', () => { + it('will make the field have a type of time if already a number', () => { + const stringTime = toDataFrame({ + fields: [ + { + name: 'proper dates', + type: FieldType.number, + values: [1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000], + }, + { name: 'A', type: FieldType.number, values: [1, 2, 3, 4, 5] }, + ], + }); + + expect(ensureTimeField(stringTime.fields[0])).toEqual({ + config: {}, + name: 'proper dates', + type: FieldType.time, + values: new ArrayVector([1626674400000, 1627020000000, 1627192800000, 1627797600000, 1627884000000]), + }); + }); +}); diff --git a/packages/grafana-data/src/transformations/transformers/convertFieldType.ts b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts new file mode 100644 index 00000000000..277e1db3322 --- /dev/null +++ b/packages/grafana-data/src/transformations/transformers/convertFieldType.ts @@ -0,0 +1,170 @@ +import { SynchronousDataTransformerInfo } from '../../types'; +import { map } from 'rxjs/operators'; + +import { DataTransformerID } from './ids'; +import { DataFrame, Field, FieldType } from '../../types/dataFrame'; +import { dateTimeParse } from '../../datetime'; +import { ArrayVector } from '../../vector'; + +export interface ConvertFieldTypeTransformerOptions { + conversions: ConvertFieldTypeOptions[]; +} + +export interface ConvertFieldTypeOptions { + targetField?: string; + destinationType?: FieldType; + dateFormat?: string; +} + +/** + * @alpha + */ +export const convertFieldTypeTransformer: SynchronousDataTransformerInfo = { + id: DataTransformerID.convertFieldType, + name: 'Convert field type', + description: 'Convert a field to a specified field type', + defaultOptions: { + fields: {}, + conversions: [{ targetField: undefined, destinationType: undefined, dateFormat: undefined }], + }, + + operator: (options) => (source) => source.pipe(map((data) => convertFieldTypeTransformer.transformer(options)(data))), + + transformer: (options: ConvertFieldTypeTransformerOptions) => (data: DataFrame[]) => { + if (!Array.isArray(data) || data.length === 0) { + return data; + } + const timeParsed = convertFieldTypes(options, data); + if (!timeParsed) { + return []; + } + return timeParsed; + }, +}; + +/** + * @alpha + */ +export function convertFieldTypes(options: ConvertFieldTypeTransformerOptions, frames: DataFrame[]): DataFrame[] { + if (!options.conversions.length) { + return frames; + } + + const frameCopy: DataFrame[] = []; + + frames.forEach((frame) => { + for (let fieldIdx = 0; fieldIdx < frame.fields.length; fieldIdx++) { + let field = frame.fields[fieldIdx]; + for (let cIdx = 0; cIdx < options.conversions.length; cIdx++) { + if (field.name === options.conversions[cIdx].targetField) { + //check in about matchers with Ryan + const conversion = options.conversions[cIdx]; + frame.fields[fieldIdx] = convertFieldType(field, conversion); + break; + } + } + } + frameCopy.push(frame); + }); + return frameCopy; +} + +export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): Field { + switch (opts.destinationType) { + case FieldType.time: + return ensureTimeField(field, opts.dateFormat); + case FieldType.number: + return fieldToNumberField(field); + case FieldType.string: + return fieldToStringField(field); + case FieldType.boolean: + return fieldToBooleanField(field); + default: + return field; + } +} + +export function fieldToTimeField(field: Field, dateFormat?: string): Field { + let opts = dateFormat ? { format: dateFormat } : undefined; + + const timeValues = field.values.toArray().slice(); + + for (let t = 0; t < timeValues.length; t++) { + if (timeValues[t]) { + let parsed = dateTimeParse(timeValues[t], opts).valueOf(); + timeValues[t] = Number.isFinite(parsed) ? parsed : null; + } else { + timeValues[t] = null; + } + } + + return { + ...field, + type: FieldType.time, + values: new ArrayVector(timeValues), + }; +} + +function fieldToNumberField(field: Field): Field { + const numValues = field.values.toArray().slice(); + + for (let n = 0; n < numValues.length; n++) { + if (numValues[n]) { + let number = +numValues[n]; + numValues[n] = Number.isFinite(number) ? number : null; + } else { + numValues[n] = null; + } + } + + return { + ...field, + type: FieldType.number, + values: new ArrayVector(numValues), + }; +} + +function fieldToBooleanField(field: Field): Field { + const booleanValues = field.values.toArray().slice(); + + for (let b = 0; b < booleanValues.length; b++) { + booleanValues[b] = Boolean(booleanValues[b]); + } + + return { + ...field, + type: FieldType.boolean, + values: new ArrayVector(booleanValues), + }; +} + +function fieldToStringField(field: Field): Field { + const stringValues = field.values.toArray().slice(); + + for (let s = 0; s < stringValues.length; s++) { + stringValues[s] = `${stringValues[s]}`; + } + + return { + ...field, + type: FieldType.string, + values: new ArrayVector(stringValues), + }; +} + +/** + * @alpha + */ +export function ensureTimeField(field: Field, dateFormat?: string): Field { + const firstValueTypeIsNumber = typeof field.values.get(0) === 'number'; + if (field.type === FieldType.time && firstValueTypeIsNumber) { + return field; //already time + } + if (firstValueTypeIsNumber) { + return { + ...field, + type: FieldType.time, //assumes it should be time + }; + } + return fieldToTimeField(field, dateFormat); +} diff --git a/packages/grafana-data/src/transformations/transformers/ids.ts b/packages/grafana-data/src/transformations/transformers/ids.ts index a09e2075e22..67f9a8cdc4c 100644 --- a/packages/grafana-data/src/transformations/transformers/ids.ts +++ b/packages/grafana-data/src/transformations/transformers/ids.ts @@ -26,4 +26,5 @@ export enum DataTransformerID { configFromData = 'configFromData', rowsToFields = 'rowsToFields', prepareTimeSeries = 'prepareTimeSeries', + convertFieldType = 'convertFieldType', } diff --git a/packages/grafana-ui/src/components/MatchersUI/FieldNamePicker.tsx b/packages/grafana-ui/src/components/MatchersUI/FieldNamePicker.tsx index 6f6abc6b9ae..a3a28db2c27 100644 --- a/packages/grafana-ui/src/components/MatchersUI/FieldNamePicker.tsx +++ b/packages/grafana-ui/src/components/MatchersUI/FieldNamePicker.tsx @@ -30,9 +30,11 @@ export const FieldNamePicker: React.FC {settings.info && } diff --git a/packages/grafana-ui/src/components/uPlot/utils.ts b/packages/grafana-ui/src/components/uPlot/utils.ts index 8ffb02866e0..59ac16919bb 100755 --- a/packages/grafana-ui/src/components/uPlot/utils.ts +++ b/packages/grafana-ui/src/components/uPlot/utils.ts @@ -1,4 +1,4 @@ -import { DataFrame, dateTime, Field, FieldType } from '@grafana/data'; +import { DataFrame, ensureTimeField, Field, FieldType } from '@grafana/data'; import { StackingMode } from '@grafana/schema'; import { createLogger } from '../../utils/logger'; import { attachDebugger } from '../../utils'; @@ -48,16 +48,7 @@ export function preparePlotData(frame: DataFrame, onStackMeta?: (meta: StackMeta const f = frame.fields[i]; if (f.type === FieldType.time) { - if (f.values.length > 0 && typeof f.values.get(0) === 'string') { - const timestamps = []; - for (let i = 0; i < f.values.length; i++) { - timestamps.push(dateTime(f.values.get(i)).valueOf()); - } - result.push(timestamps); - seriesIndex++; - continue; - } - result.push(f.values.toArray()); + result.push(ensureTimeField(f).values.toArray()); seriesIndex++; continue; } diff --git a/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx b/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx new file mode 100644 index 00000000000..217e5e95f3c --- /dev/null +++ b/public/app/core/components/TransformersUI/ConvertFieldTypeTransformerEditor.tsx @@ -0,0 +1,149 @@ +import React, { useCallback } from 'react'; +import { + DataTransformerID, + FieldNamePickerConfigSettings, + FieldType, + SelectableValue, + StandardEditorsRegistryItem, + standardTransformers, + TransformerRegistryItem, + TransformerUIProps, +} from '@grafana/data'; + +import { ConvertFieldTypeTransformerOptions } from '@grafana/data/src/transformations/transformers/convertFieldType'; +import { Button, InlineField, InlineFieldRow, Input, Select } from '@grafana/ui'; +import { FieldNamePicker } from '../../../../../packages/grafana-ui/src/components/MatchersUI/FieldNamePicker'; +import { ConvertFieldTypeOptions } from '../../../../../packages/grafana-data/src/transformations/transformers/convertFieldType'; + +const fieldNamePickerSettings: StandardEditorsRegistryItem = { + settings: { width: 24 }, +} as any; + +export const ConvertFieldTypeTransformerEditor: React.FC> = ({ + input, + options, + onChange, +}) => { + const allTypes: Array> = [ + { value: FieldType.number, label: 'Numeric' }, + { value: FieldType.string, label: 'String' }, + { value: FieldType.time, label: 'Time' }, + { value: FieldType.boolean, label: 'Boolean' }, + ]; + + const onSelectField = useCallback( + (idx) => (value: string | undefined) => { + const conversions = options.conversions; + conversions[idx] = { ...conversions[idx], targetField: value ?? '' }; + onChange({ + ...options, + conversions: conversions, + }); + }, + [onChange, options] + ); + + const onSelectDestinationType = useCallback( + (idx) => (value: SelectableValue) => { + const conversions = options.conversions; + conversions[idx] = { ...conversions[idx], destinationType: value.value }; + onChange({ + ...options, + conversions: conversions, + }); + }, + [onChange, options] + ); + + const onInputFormat = useCallback( + (idx) => (value: SelectableValue) => { + const conversions = options.conversions; + conversions[idx] = { ...conversions[idx], dateFormat: value.value }; + onChange({ + ...options, + conversions: conversions, + }); + }, + [onChange, options] + ); + + const onAddConvertFieldType = useCallback(() => { + onChange({ + ...options, + conversions: [ + ...options.conversions, + { targetField: undefined, destinationType: undefined, dateFormat: undefined }, + ], + }); + }, [onChange, options]); + + const onRemoveConvertFieldType = useCallback( + (idx) => { + const removed = options.conversions; + removed.splice(idx, 1); + onChange({ + ...options, + conversions: removed, + }); + }, + [onChange, options] + ); + + return ( + <> + {options.conversions.map((c: ConvertFieldTypeOptions, idx: number) => { + return ( + + + + + + + + )} + + + ); +}; + +export const convertFieldTypeTransformRegistryItem: TransformerRegistryItem = { + id: DataTransformerID.convertFieldType, + editor: ConvertFieldTypeTransformerEditor, + transformation: standardTransformers.convertFieldTypeTransformer, + name: standardTransformers.convertFieldTypeTransformer.name, + description: standardTransformers.convertFieldTypeTransformer.description, +}; diff --git a/public/app/core/utils/standardTransformers.ts b/public/app/core/utils/standardTransformers.ts index 16f8b0a67d5..1e2002e4754 100644 --- a/public/app/core/utils/standardTransformers.ts +++ b/public/app/core/utils/standardTransformers.ts @@ -17,6 +17,7 @@ import { histogramTransformRegistryItem } from '../components/TransformersUI/His import { rowsToFieldsTransformRegistryItem } from '../components/TransformersUI/rowsToFields/RowsToFieldsTransformerEditor'; import { configFromQueryTransformRegistryItem } from '../components/TransformersUI/configFromQuery/ConfigFromQueryTransformerEditor'; import { prepareTimeseriesTransformerRegistryItem } from '../components/TransformersUI/prepareTimeSeries/PrepareTimeSeriesEditor'; +import { convertFieldTypeTransformRegistryItem } from '../components/TransformersUI/ConvertFieldTypeTransformerEditor'; export const getStandardTransformers = (): Array> => { return [ @@ -38,5 +39,6 @@ export const getStandardTransformers = (): Array> = rowsToFieldsTransformRegistryItem, configFromQueryTransformRegistryItem, prepareTimeseriesTransformerRegistryItem, + convertFieldTypeTransformRegistryItem, ]; };