Transformations: Add support for setting timezone in Format time and Convert field type transformations (#76316)

* Better TZ support.

* Clean things up

* Update convertFieldType

* Update tooltips

* Remove unecessary code

* Prettier

* Add internal TZs

* Prettier

* Fix tests

* Make sure tests expect time with timezone

* Use true lol

* Centralize logic for options
This commit is contained in:
Kyle Cunningham 2023-10-13 01:20:00 -05:00 committed by GitHub
parent d5945bc26e
commit 1a34cf6b0e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 132 additions and 81 deletions

View File

@ -1,6 +1,8 @@
import { map } from 'rxjs/operators'; import { map } from 'rxjs/operators';
import { dateTimeParse } from '../../datetime'; import { TimeZone } from '@grafana/schema';
import { DateTimeOptionsWhenParsing, dateTimeParse } from '../../datetime';
import { SynchronousDataTransformerInfo } from '../../types'; import { SynchronousDataTransformerInfo } from '../../types';
import { DataFrame, EnumFieldConfig, Field, FieldType } from '../../types/dataFrame'; import { DataFrame, EnumFieldConfig, Field, FieldType } from '../../types/dataFrame';
import { fieldMatchers } from '../matchers'; import { fieldMatchers } from '../matchers';
@ -25,8 +27,13 @@ export interface ConvertFieldTypeOptions {
* Date format to parse a string datetime * Date format to parse a string datetime
*/ */
dateFormat?: string; dateFormat?: string;
/**
/** When converting to an enumeration, this is the target config */ * When converting a date to a string an option timezone.
*/
timezone?: TimeZone;
/**
* When converting to an enumeration, this is the target config
*/
enumConfig?: EnumFieldConfig; enumConfig?: EnumFieldConfig;
} }
@ -36,7 +43,7 @@ export const convertFieldTypeTransformer: SynchronousDataTransformerInfo<Convert
description: 'Convert a field to a specified field type.', description: 'Convert a field to a specified field type.',
defaultOptions: { defaultOptions: {
fields: {}, fields: {},
conversions: [{ targetField: undefined, destinationType: undefined, dateFormat: undefined }], conversions: [{ targetField: undefined, destinationType: undefined, dateFormat: undefined, timezone: undefined }],
}, },
operator: (options, ctx) => (source) => operator: (options, ctx) => (source) =>
@ -96,7 +103,7 @@ export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): F
case FieldType.number: case FieldType.number:
return fieldToNumberField(field); return fieldToNumberField(field);
case FieldType.string: case FieldType.string:
return fieldToStringField(field, opts.dateFormat); return fieldToStringField(field, opts.dateFormat, { timeZone: opts.timezone });
case FieldType.boolean: case FieldType.boolean:
return fieldToBooleanField(field); return fieldToBooleanField(field);
case FieldType.enum: case FieldType.enum:
@ -179,12 +186,19 @@ function fieldToBooleanField(field: Field): Field {
}; };
} }
function fieldToStringField(field: Field, dateFormat?: string): Field { /**
* @internal
*/
export function fieldToStringField(
field: Field,
dateFormat?: string,
parseOptions?: DateTimeOptionsWhenParsing
): Field {
let values = field.values; let values = field.values;
switch (field.type) { switch (field.type) {
case FieldType.time: case FieldType.time:
values = values.map((v) => dateTimeParse(v).format(dateFormat)); values = values.map((v) => dateTimeParse(v, parseOptions).format(dateFormat));
break; break;
case FieldType.other: case FieldType.other:

View File

@ -13,10 +13,10 @@ describe('Format Time Transformer', () => {
const options = { const options = {
timeField: 'time', timeField: 'time',
outputFormat: 'YYYY-MM', outputFormat: 'YYYY-MM',
useTimezone: false, timezone: 'utc',
}; };
const formatter = createTimeFormatter(options.timeField, options.outputFormat, options.useTimezone); const formatter = createTimeFormatter(options.timeField, options.outputFormat, options.timezone);
const frame = toDataFrame({ const frame = toDataFrame({
fields: [ fields: [
{ {
@ -35,10 +35,10 @@ describe('Format Time Transformer', () => {
const options = { const options = {
timeField: 'time', timeField: 'time',
outputFormat: 'YYYY-MM h:mm:ss a', outputFormat: 'YYYY-MM h:mm:ss a',
useTimezone: false, timezone: 'utc',
}; };
const formatter = createTimeFormatter(options.timeField, options.outputFormat, options.useTimezone); const formatter = createTimeFormatter(options.timeField, options.outputFormat, options.timezone);
const frame = toDataFrame({ const frame = toDataFrame({
fields: [ fields: [
{ {
@ -51,11 +51,11 @@ describe('Format Time Transformer', () => {
const newFrame = formatter(frame.fields); const newFrame = formatter(frame.fields);
expect(newFrame[0].values).toEqual([ expect(newFrame[0].values).toEqual([
'2021-02 1:46:40 am', '2021-02 6:46:40 am',
'2023-07 2:00:00 pm', '2023-07 8:00:00 pm',
'2023-04 3:20:00 pm', '2023-04 9:20:00 pm',
'2023-07 5:34:49 pm', '2023-07 11:34:49 pm',
'2023-08 3:20:00 pm', '2023-08 9:20:00 pm',
]); ]);
}); });
@ -63,10 +63,10 @@ describe('Format Time Transformer', () => {
const options = { const options = {
timeField: 'time', timeField: 'time',
outputFormat: 'YYYY-MM h:mm:ss a', outputFormat: 'YYYY-MM h:mm:ss a',
useTimezone: false, timezone: 'utc',
}; };
const formatter = createTimeFormatter(options.timeField, options.outputFormat, options.useTimezone); const formatter = createTimeFormatter(options.timeField, options.outputFormat, options.timezone);
const frame = toDataFrame({ const frame = toDataFrame({
fields: [ fields: [
{ {
@ -79,10 +79,10 @@ describe('Format Time Transformer', () => {
const newFrame = formatter(frame.fields); const newFrame = formatter(frame.fields);
expect(newFrame[0].values).toEqual([ expect(newFrame[0].values).toEqual([
'2021-02 1:46:40 am', '2021-02 6:46:40 am',
'2023-07 2:00:00 pm', '2023-07 8:00:00 pm',
'2023-04 3:20:00 pm', '2023-04 9:20:00 pm',
'2023-07 5:34:49 pm', '2023-07 11:34:49 pm',
'Invalid date', 'Invalid date',
]); ]);
}); });

View File

@ -1,16 +1,17 @@
import moment from 'moment-timezone';
import { map } from 'rxjs/operators'; import { map } from 'rxjs/operators';
import { getTimeZone, getTimeZoneInfo } from '../../datetime'; import { TimeZone } from '@grafana/schema';
import { Field, FieldType } from '../../types';
import { Field } from '../../types';
import { DataTransformerInfo } from '../../types/transformations'; import { DataTransformerInfo } from '../../types/transformations';
import { fieldToStringField } from './convertFieldType';
import { DataTransformerID } from './ids'; import { DataTransformerID } from './ids';
export interface FormatTimeTransformerOptions { export interface FormatTimeTransformerOptions {
timeField: string; timeField: string;
outputFormat: string; outputFormat: string;
useTimezone: boolean; timezone: TimeZone;
} }
export const formatTimeTransformer: DataTransformerInfo<FormatTimeTransformerOptions> = { export const formatTimeTransformer: DataTransformerInfo<FormatTimeTransformerOptions> = {
@ -23,7 +24,7 @@ export const formatTimeTransformer: DataTransformerInfo<FormatTimeTransformerOpt
map((data) => { map((data) => {
// If a field and a format are configured // If a field and a format are configured
// then format the time output // then format the time output
const formatter = createTimeFormatter(options.timeField, options.outputFormat, options.useTimezone); const formatter = createTimeFormatter(options.timeField, options.outputFormat, options.timezone);
if (!Array.isArray(data) || data.length === 0) { if (!Array.isArray(data) || data.length === 0) {
return data; return data;
@ -40,37 +41,21 @@ export const formatTimeTransformer: DataTransformerInfo<FormatTimeTransformerOpt
/** /**
* @internal * @internal
*/ */
export const createTimeFormatter = export const createTimeFormatter = (timeField: string, outputFormat: string, timezone: string) => (fields: Field[]) => {
(timeField: string, outputFormat: string, useTimezone: boolean) => (fields: Field[]) => { return fields.map((field) => {
const tz = getTimeZone(); // Find the configured field
if (field.name === timeField) {
return fields.map((field) => { // Update values to use the configured format
// Find the configured field let formattedField = null;
if (field.name === timeField) { if (timezone) {
// Update values to use the configured format formattedField = fieldToStringField(field, outputFormat, { timeZone: timezone });
const newVals = field.values.map((value) => { } else {
const date = moment(value); formattedField = fieldToStringField(field, outputFormat);
// Apply configured timezone if the
// option has been set. Otherwise
// use the date directly
if (useTimezone) {
const info = getTimeZoneInfo(tz, value);
const realTz = info !== undefined ? info.ianaName : 'UTC';
return date.tz(realTz).format(outputFormat);
} else {
return date.format(outputFormat);
}
});
return {
...field,
type: FieldType.string,
values: newVals,
};
} }
return field; return formattedField;
}); }
};
return field;
});
};

View File

@ -10,6 +10,7 @@ import {
TransformerRegistryItem, TransformerRegistryItem,
TransformerUIProps, TransformerUIProps,
TransformerCategory, TransformerCategory,
getTimeZones,
} from '@grafana/data'; } from '@grafana/data';
import { import {
ConvertFieldTypeOptions, ConvertFieldTypeOptions,
@ -21,6 +22,8 @@ import { allFieldTypeIconOptions } from '@grafana/ui/src/components/MatchersUI/F
import { hasAlphaPanels } from 'app/core/config'; import { hasAlphaPanels } from 'app/core/config';
import { findField } from 'app/features/dimensions'; import { findField } from 'app/features/dimensions';
import { getTimezoneOptions } from '../utils';
const fieldNamePickerSettings = { const fieldNamePickerSettings = {
settings: { width: 24, isClearable: false }, settings: { width: 24, isClearable: false },
} as StandardEditorsRegistryItem<string, FieldNamePickerConfigSettings>; } as StandardEditorsRegistryItem<string, FieldNamePickerConfigSettings>;
@ -31,6 +34,15 @@ export const ConvertFieldTypeTransformerEditor = ({
onChange, onChange,
}: TransformerUIProps<ConvertFieldTypeTransformerOptions>) => { }: TransformerUIProps<ConvertFieldTypeTransformerOptions>) => {
const allTypes = allFieldTypeIconOptions.filter((v) => v.value !== FieldType.trace); const allTypes = allFieldTypeIconOptions.filter((v) => v.value !== FieldType.trace);
const timeZoneOptions: Array<SelectableValue<string>> = getTimezoneOptions(true);
// Format timezone options
const tzs = getTimeZones();
timeZoneOptions.push({ label: 'Browser', value: 'browser' });
timeZoneOptions.push({ label: 'UTC', value: 'utc' });
for (const tz of tzs) {
timeZoneOptions.push({ label: tz, value: tz });
}
const onSelectField = useCallback( const onSelectField = useCallback(
(idx: number) => (value: string | undefined) => { (idx: number) => (value: string | undefined) => {
@ -90,6 +102,18 @@ export const ConvertFieldTypeTransformerEditor = ({
[onChange, options] [onChange, options]
); );
const onTzChange = useCallback(
(idx: number) => (value: SelectableValue<string>) => {
const conversions = options.conversions;
conversions[idx] = { ...conversions[idx], timezone: value?.value };
onChange({
...options,
conversions: conversions,
});
},
[onChange, options]
);
return ( return (
<> <>
{options.conversions.map((c: ConvertFieldTypeOptions, idx: number) => { {options.conversions.map((c: ConvertFieldTypeOptions, idx: number) => {
@ -128,14 +152,19 @@ export const ConvertFieldTypeTransformerEditor = ({
)} )}
{c.destinationType === FieldType.string && {c.destinationType === FieldType.string &&
(c.dateFormat || findField(input?.[0], c.targetField)?.type === FieldType.time) && ( (c.dateFormat || findField(input?.[0], c.targetField)?.type === FieldType.time) && (
<InlineField label="Date format" tooltip="Specify the output format."> <>
<Input <InlineField label="Date format" tooltip="Specify the output format.">
value={c.dateFormat} <Input
placeholder={'e.g. YYYY-MM-DD'} value={c.dateFormat}
onChange={onInputFormat(idx)} placeholder={'e.g. YYYY-MM-DD'}
width={24} onChange={onInputFormat(idx)}
/> width={24}
</InlineField> />
</InlineField>
<InlineField label="Set timezone" tooltip="Set the timezone of the date manually">
<Select options={timeZoneOptions} value={c.timezone} onChange={onTzChange(idx)} isClearable />
</InlineField>
</>
)} )}
<Button <Button
size="md" size="md"

View File

@ -10,7 +10,9 @@ import {
PluginState, PluginState,
} from '@grafana/data'; } from '@grafana/data';
import { FormatTimeTransformerOptions } from '@grafana/data/src/transformations/transformers/formatTime'; import { FormatTimeTransformerOptions } from '@grafana/data/src/transformations/transformers/formatTime';
import { Select, InlineFieldRow, InlineField, Input, InlineSwitch } from '@grafana/ui'; import { Select, InlineFieldRow, InlineField, Input } from '@grafana/ui';
import { getTimezoneOptions } from '../utils';
export function FormatTimeTransfomerEditor({ export function FormatTimeTransfomerEditor({
input, input,
@ -18,6 +20,7 @@ export function FormatTimeTransfomerEditor({
onChange, onChange,
}: TransformerUIProps<FormatTimeTransformerOptions>) { }: TransformerUIProps<FormatTimeTransformerOptions>) {
const timeFields: Array<SelectableValue<string>> = []; const timeFields: Array<SelectableValue<string>> = [];
const timeZoneOptions: Array<SelectableValue<string>> = getTimezoneOptions(true);
// Get time fields // Get time fields
for (const frame of input) { for (const frame of input) {
@ -51,12 +54,16 @@ export function FormatTimeTransfomerEditor({
[onChange, options] [onChange, options]
); );
const onUseTzChange = useCallback(() => { const onTzChange = useCallback(
onChange({ (value: SelectableValue<string>) => {
...options, const val = value?.value !== undefined ? value.value : '';
useTimezone: !options.useTimezone, onChange({
}); ...options,
}, [onChange, options]); timezone: val,
});
},
[onChange, options]
);
return ( return (
<> <>
@ -87,12 +94,8 @@ export function FormatTimeTransfomerEditor({
> >
<Input onChange={onFormatChange} value={options.outputFormat} /> <Input onChange={onFormatChange} value={options.outputFormat} />
</InlineField> </InlineField>
<InlineField <InlineField label="Set Timezone" tooltip="Set the timezone of the date manually" labelWidth={20}>
label="Use Timezone" <Select options={timeZoneOptions} value={options.timezone} onChange={onTzChange} isClearable />
tooltip="Use the user's configured timezone when formatting time."
labelWidth={20}
>
<InlineSwitch value={options.useTimezone} transparent={true} onChange={onUseTzChange} />
</InlineField> </InlineField>
</InlineFieldRow> </InlineFieldRow>
</> </>

View File

@ -1,6 +1,6 @@
import { useMemo } from 'react'; import { useMemo } from 'react';
import { DataFrame, getFieldDisplayName, TransformerCategory } from '@grafana/data'; import { DataFrame, getFieldDisplayName, TransformerCategory, SelectableValue, getTimeZones } from '@grafana/data';
import { config } from '@grafana/runtime'; import { config } from '@grafana/runtime';
export function useAllFieldNamesFromDataFrames(input: DataFrame[]): string[] { export function useAllFieldNamesFromDataFrames(input: DataFrame[]): string[] {
@ -61,3 +61,23 @@ export const numberOrVariableValidator = (value: string | number) => {
} }
return false; return false;
}; };
export function getTimezoneOptions(includeInternal: boolean) {
const timeZoneOptions: Array<SelectableValue<string>> = [];
// There are currently only two internal timezones
// Browser and UTC. We add the manually to avoid
// funky string manipulation.
if (includeInternal) {
timeZoneOptions.push({ label: 'Browser', value: 'browser' });
timeZoneOptions.push({ label: 'UTC', value: 'utc' });
}
// Add all other timezones
const tzs = getTimeZones();
for (const tz of tzs) {
timeZoneOptions.push({ label: tz, value: tz });
}
return timeZoneOptions;
}