TimeSeries: insert null values at each missing interval (#49036)

This commit is contained in:
Leon Sorokin 2022-05-16 22:13:27 -05:00 committed by GitHub
parent 14c2f4f3da
commit eecdbebf75
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 41 additions and 13 deletions

View File

@ -59,9 +59,9 @@ describe('nullInsertThreshold Transformer', () => {
const result = applyNullInsertThreshold(df);
expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 10]);
expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, 8]);
expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, 'c']);
expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, null, null, null, null, null, 8]);
expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, null, null, null, null, null, 'c']);
});
test('should insert nulls at +threshold between adjacent > threshold: 2', () => {
@ -93,9 +93,9 @@ describe('nullInsertThreshold Transformer', () => {
const result = applyNullInsertThreshold(df);
expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 10]);
expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, 8]);
expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, 'c']);
expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, null, null, null, null, null, 8]);
expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, null, null, null, null, null, 'c']);
});
test('should insert trailing null at end +interval when timeRange.to.valueOf() exceeds threshold', () => {
@ -110,9 +110,21 @@ describe('nullInsertThreshold Transformer', () => {
const result = applyNullInsertThreshold(df, null, 13);
expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 10, 11]);
expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, 8, null]);
expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, 'c', null]);
expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]);
expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, null, null, null, null, null, 8, null]);
expect(result.fields[2].values.toArray()).toStrictEqual([
'a',
null,
'b',
null,
null,
null,
null,
null,
null,
'c',
null,
]);
// should work for frames with 1 datapoint
const df2 = new MutableDataFrame({

View File

@ -13,7 +13,8 @@ export function applyNullInsertThreshold(
frame: DataFrame,
refFieldName?: string | null,
refFieldPseudoMax: number | null = null,
insertMode: InsertMode = INSERT_MODES.threshold
insertMode: InsertMode = INSERT_MODES.threshold,
thorough = true
): DataFrame {
if (frame.length === 0) {
return frame;
@ -49,7 +50,14 @@ export function applyNullInsertThreshold(
const frameValues = frame.fields.map((field) => field.values.toArray());
const filledFieldValues = nullInsertThreshold(refValues, frameValues, threshold, refFieldPseudoMax, insertMode);
const filledFieldValues = nullInsertThreshold(
refValues,
frameValues,
threshold,
refFieldPseudoMax,
insertMode,
thorough
);
if (filledFieldValues === frameValues) {
return frame;
@ -77,7 +85,9 @@ function nullInsertThreshold(
threshold: number,
// will insert a trailing null when refFieldPseudoMax > last datapoint + threshold
refFieldPseudoMax: number | null = null,
getInsertValue: InsertMode
getInsertValue: InsertMode,
// will insert the value at every missing interval
thorough: boolean
) {
const len = refValues.length;
let prevValue: number = refValues[0];
@ -86,8 +96,14 @@ function nullInsertThreshold(
for (let i = 1; i < len; i++) {
const curValue = refValues[i];
if (curValue - prevValue > threshold) {
while (curValue - prevValue > threshold) {
refValuesNew.push(getInsertValue(prevValue, curValue, threshold));
prevValue += threshold;
if (!thorough) {
break;
}
}
refValuesNew.push(curValue);