Transformers: ensure ISO 8601 date strings are not parsed in local TZ (#41835)

This commit is contained in:
Leon Sorokin 2021-11-17 14:56:04 -06:00 committed by GitHub
parent e237ff20a9
commit 6844cf57a0
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 37 additions and 3 deletions

View File

@ -1,8 +1,14 @@
import { toDataFrame } from '../../dataframe/processDataFrame';
import { FieldType } from '../../types/dataFrame';
import { Field, FieldType } from '../../types/dataFrame';
import { mockTransformationsRegistry } from '../../utils/tests/mockTransformationsRegistry';
import { ArrayVector } from '../../vector';
import { ensureTimeField, convertFieldType, convertFieldTypes, convertFieldTypeTransformer } from './convertFieldType';
import {
ensureTimeField,
convertFieldType,
convertFieldTypes,
convertFieldTypeTransformer,
fieldToTimeField,
} from './convertFieldType';
describe('field convert type', () => {
it('will parse properly formatted strings to time', () => {
@ -233,3 +239,24 @@ describe('ensureTimeField', () => {
});
});
});
describe('fieldToTimeField', () => {
// this needs to run in a non-UTC timezone env to ensure the parsing is not dependent on env tz settings
//process.env.TZ = 'Pacific/Easter';
it('should always parse ISO 8601 date strings in UTC timezone (e.g. 2011-10-05T14:48:00.000Z)', () => {
const stringTimeField: Field = {
config: {},
name: 'ISO 8601 date strings',
type: FieldType.time,
values: new ArrayVector(['2021-11-11T19:45:00.000Z']),
};
expect(fieldToTimeField(stringTimeField)).toEqual({
config: {},
name: 'ISO 8601 date strings',
type: FieldType.time,
values: new ArrayVector([1636659900000]),
});
});
});

View File

@ -104,6 +104,9 @@ export function convertFieldType(field: Field, opts: ConvertFieldTypeOptions): F
}
}
// matches ISO 8601, e.g. 2021-11-11T19:45:00.000Z (float portion optional)
const iso8601Regex = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{3})?Z$/;
/**
* @internal
*/
@ -112,9 +115,13 @@ export function fieldToTimeField(field: Field, dateFormat?: string): Field {
const timeValues = field.values.toArray().slice();
let firstDefined = timeValues.find((v) => v != null);
let isISO8601 = typeof firstDefined === 'string' && iso8601Regex.test(firstDefined);
for (let t = 0; t < timeValues.length; t++) {
if (timeValues[t]) {
let parsed = dateTimeParse(timeValues[t], opts).valueOf();
let parsed = isISO8601 ? Date.parse(timeValues[t]) : dateTimeParse(timeValues[t], opts).valueOf();
timeValues[t] = Number.isFinite(parsed) ? parsed : null;
} else {
timeValues[t] = null;