State Timeline: Fix Null Value Filling and Value Transformation (#50054)

packages/grafana-data/src/types/dataFrame.ts

@@ -181,6 +181,13 @@ export interface FieldState {
    * This is only related to the cached displayName property above.
    */
   multipleFrames?: boolean;
+
+  /**
+   * Boolean value is true if a null filling threshold has been applied
+   * against the frame of the field. This is used to avoid cases in which
+   * this would applied more than one time.
+   */
+  nullThresholdApplied?: boolean;
 }
 
 /** @public */

packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.test.ts

@@ -57,7 +57,7 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df);
+    const result = applyNullInsertThreshold({ frame: df });
 
     expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
     expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, null, null, null, null, null, 8]);
@@ -74,7 +74,7 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df);
+    const result = applyNullInsertThreshold({ frame: df });
 
     expect(result.fields[0].values.toArray()).toStrictEqual([5, 7, 9, 11]);
     expect(result.fields[1].values.toArray()).toStrictEqual([4, 6, null, 8]);
@@ -91,13 +91,63 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df);
+    const result = applyNullInsertThreshold({ frame: df });
 
     expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
     expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, null, null, null, null, null, 8]);
     expect(result.fields[2].values.toArray()).toStrictEqual(['a', null, 'b', null, null, null, null, null, null, 'c']);
   });
 
+  test('should insert leading null at beginning +interval when timeRange.from.valueOf() exceeds threshold', () => {
+    const df = new MutableDataFrame({
+      refId: 'A',
+      fields: [
+        { name: 'Time', type: FieldType.time, config: { interval: 1 }, values: [4, 6, 13] },
+        { name: 'One', type: FieldType.number, values: [4, 6, 8] },
+        { name: 'Two', type: FieldType.string, values: ['a', 'b', 'c'] },
+      ],
+    });
+
+    const result = applyNullInsertThreshold({
+      frame: df,
+      refFieldName: null,
+      refFieldPseudoMin: 1,
+      refFieldPseudoMax: 13,
+    });
+
+    expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]);
+    expect(result.fields[1].values.toArray()).toStrictEqual([
+      null,
+      null,
+      null,
+      4,
+      null,
+      6,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      8,
+    ]);
+    expect(result.fields[2].values.toArray()).toStrictEqual([
+      null,
+      null,
+      null,
+      'a',
+      null,
+      'b',
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      'c',
+    ]);
+  });
+
   test('should insert trailing null at end +interval when timeRange.to.valueOf() exceeds threshold', () => {
     const df = new MutableDataFrame({
       refId: 'A',
@@ -108,10 +158,24 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df, null, 13);
+    const result = applyNullInsertThreshold({ frame: df, refFieldName: null, refFieldPseudoMax: 13 });
 
-    expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]);
-    expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, null, null, null, null, null, 8, null]);
+    expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]);
+    expect(result.fields[1].values.toArray()).toStrictEqual([
+      4,
+      null,
+      6,
+      null,
+      null,
+      null,
+      null,
+      null,
+      null,
+      8,
+      null,
+      null,
+      null,
+    ]);
     expect(result.fields[2].values.toArray()).toStrictEqual([
       'a',
       null,
@@ -124,6 +188,8 @@ describe('nullInsertThreshold Transformer', () => {
       null,
       'c',
       null,
+      null,
+      null,
     ]);
 
     // should work for frames with 1 datapoint
@@ -136,7 +202,9 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result2 = applyNullInsertThreshold(df2, null, 13);
+    // Max is 2 as opposed to the above 13 otherwise
+    // we get 12 nulls instead of the additional 1
+    const result2 = applyNullInsertThreshold({ frame: df2, refFieldName: null, refFieldPseudoMax: 2 });
 
     expect(result2.fields[0].values.toArray()).toStrictEqual([1, 2]);
     expect(result2.fields[1].values.toArray()).toStrictEqual([1, null]);
@@ -154,7 +222,7 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df);
+    const result = applyNullInsertThreshold({ frame: df });
 
     expect(result.fields[0].values.toArray()).toStrictEqual([5, 6, 7, 8, 11]);
     expect(result.fields[1].values.toArray()).toStrictEqual([4, null, 6, null, 8]);
@@ -170,7 +238,7 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df);
+    const result = applyNullInsertThreshold({ frame: df });
 
     expect(result).toBe(df);
   });
@@ -184,7 +252,7 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df);
+    const result = applyNullInsertThreshold({ frame: df });
 
     expect(result).toBe(df);
   });
@@ -198,7 +266,7 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df);
+    const result = applyNullInsertThreshold({ frame: df });
 
     expect(result).toBe(df);
   });
@@ -212,7 +280,7 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df);
+    const result = applyNullInsertThreshold({ frame: df });
 
     expect(result).toBe(df);
   });
@@ -226,7 +294,7 @@ describe('nullInsertThreshold Transformer', () => {
       ],
     });
 
-    const result = applyNullInsertThreshold(df, 'Time2');
+    const result = applyNullInsertThreshold({ frame: df, refFieldName: 'Time2' });
 
     expect(result).toBe(df);
   });
@@ -238,7 +306,7 @@ describe('nullInsertThreshold Transformer', () => {
 
     // eslint-disable-next-line no-console
     console.time('insertValues-10x3k');
-    applyNullInsertThreshold(bigFrameA);
+    applyNullInsertThreshold({ frame: bigFrameA });
     // eslint-disable-next-line no-console
     console.timeEnd('insertValues-10x3k');
   });

packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.ts

@@ -9,15 +9,24 @@ const INSERT_MODES = {
   plusone: (prev: number, next: number, threshold: number) => prev + 1,
 };
 
-export function applyNullInsertThreshold(
-  frame: DataFrame,
-  refFieldName?: string | null,
-  refFieldPseudoMax: number | null = null,
-  insertMode: InsertMode = INSERT_MODES.threshold,
-  thorough = true
-): DataFrame {
-  if (frame.length === 0) {
-    return frame;
+interface NullInsertOptions {
+  frame: DataFrame;
+  refFieldName?: string | null;
+  refFieldPseudoMax?: number;
+  refFieldPseudoMin?: number;
+  insertMode?: InsertMode;
+}
+
+export function applyNullInsertThreshold(opts: NullInsertOptions): DataFrame {
+  if (opts.frame.length === 0) {
+    return opts.frame;
+  }
+
+  let thorough = true;
+  let { frame, refFieldName, refFieldPseudoMax, refFieldPseudoMin, insertMode } = opts;
+
+  if (!insertMode) {
+    insertMode = INSERT_MODES.threshold;
   }
 
   const refField = frame.fields.find((field) => {
@@ -54,6 +63,7 @@ export function applyNullInsertThreshold(
       refValues,
       frameValues,
       threshold,
+      refFieldPseudoMin,
       refFieldPseudoMax,
       insertMode,
       thorough
@@ -83,6 +93,7 @@ function nullInsertThreshold(
   refValues: number[],
   frameValues: any[][],
   threshold: number,
+  refFieldPseudoMin: number | null = null,
   // will insert a trailing null when refFieldPseudoMax > last datapoint + threshold
   refFieldPseudoMax: number | null = null,
   getInsertValue: InsertMode,
@@ -91,8 +102,26 @@ function nullInsertThreshold(
 ) {
   const len = refValues.length;
   let prevValue: number = refValues[0];
-  const refValuesNew: number[] = [prevValue];
+  const refValuesNew: number[] = [];
 
+  // Continiuously add the threshold to the minimum value
+  // While this is less than "prevValue" which is the lowest
+  // time value in the sequence add in time frames
+  if (refFieldPseudoMin != null) {
+    let minValue = refFieldPseudoMin - threshold;
+
+    while (minValue < prevValue - threshold) {
+      let nextValue = minValue + threshold;
+      refValuesNew.push(getInsertValue(minValue, nextValue, threshold));
+      minValue = nextValue;
+    }
+  }
+
+  // Insert initial value
+  refValuesNew.push(prevValue);
+
+  // Fill nulls when a value is greater than
+  // the threshold value
   for (let i = 1; i < len; i++) {
     const curValue = refValues[i];
 
@@ -111,8 +140,12 @@ function nullInsertThreshold(
     prevValue = curValue;
   }
 
-  if (refFieldPseudoMax != null && prevValue + threshold <= refFieldPseudoMax) {
-    refValuesNew.push(getInsertValue(prevValue, refFieldPseudoMax, threshold));
+  // At the end of the sequence
+  if (refFieldPseudoMax != null) {
+    while (prevValue + threshold <= refFieldPseudoMax) {
+      refValuesNew.push(getInsertValue(prevValue, refFieldPseudoMax, threshold));
+      prevValue += threshold;
+    }
   }
 
   const filledLen = refValuesNew.length;

packages/grafana-ui/src/components/GraphNG/nullToValue.test.ts

@@ -0,0 +1,94 @@
+import { FieldType, MutableDataFrame } from '@grafana/data';
+
+import { applyNullInsertThreshold } from './nullInsertThreshold';
+import { nullToValue } from './nullToValue';
+
+describe('nullToValue Transformer', () => {
+  test('should change all nulls to configured zero value', () => {
+    const df = new MutableDataFrame({
+      refId: 'A',
+      fields: [
+        { name: 'Time', type: FieldType.time, values: [1, 3, 10] },
+        {
+          name: 'One',
+          type: FieldType.number,
+          config: { custom: { insertNulls: 1 }, noValue: '0' },
+          values: [4, 6, 8],
+        },
+        {
+          name: 'Two',
+          type: FieldType.string,
+          config: { custom: { insertNulls: 1 }, noValue: '0' },
+          values: ['a', 'b', 'c'],
+        },
+      ],
+    });
+
+    const result = nullToValue(applyNullInsertThreshold({ frame: df }));
+
+    expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+    expect(result.fields[1].values.toArray()).toStrictEqual([4, 0, 6, 0, 0, 0, 0, 0, 0, 8]);
+    expect(result.fields[2].values.toArray()).toStrictEqual(['a', 0, 'b', 0, 0, 0, 0, 0, 0, 'c']);
+  });
+
+  test('should change all nulls to configured positive value', () => {
+    const df = new MutableDataFrame({
+      refId: 'A',
+      fields: [
+        { name: 'Time', type: FieldType.time, values: [5, 7, 11] },
+        {
+          name: 'One',
+          type: FieldType.number,
+          config: { custom: { insertNulls: 2 }, noValue: '1' },
+          values: [4, 6, 8],
+        },
+        {
+          name: 'Two',
+          type: FieldType.string,
+          config: { custom: { insertNulls: 2 }, noValue: '1' },
+          values: ['a', 'b', 'c'],
+        },
+      ],
+    });
+
+    const result = nullToValue(applyNullInsertThreshold({ frame: df }));
+
+    expect(result.fields[0].values.toArray()).toStrictEqual([5, 7, 9, 11]);
+    expect(result.fields[1].values.toArray()).toStrictEqual([4, 6, 1, 8]);
+    expect(result.fields[2].values.toArray()).toStrictEqual(['a', 'b', 1, 'c']);
+  });
+
+  test('should change all nulls to configured negative value', () => {
+    const df = new MutableDataFrame({
+      refId: 'A',
+      fields: [
+        { name: 'Time', type: FieldType.time, config: { interval: 1 }, values: [1, 3, 10] },
+        { name: 'One', type: FieldType.number, config: { noValue: '-1' }, values: [4, 6, 8] },
+        { name: 'Two', type: FieldType.string, config: { noValue: '-1' }, values: ['a', 'b', 'c'] },
+      ],
+    });
+
+    const result = nullToValue(applyNullInsertThreshold({ frame: df }));
+
+    expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
+    expect(result.fields[1].values.toArray()).toStrictEqual([4, -1, 6, -1, -1, -1, -1, -1, -1, 8]);
+    expect(result.fields[2].values.toArray()).toStrictEqual(['a', -1, 'b', -1, -1, -1, -1, -1, -1, 'c']);
+  });
+
+  test('should have no effect without nulls', () => {
+    const df = new MutableDataFrame({
+      refId: 'A',
+      fields: [
+        { name: 'Time', type: FieldType.time, config: { interval: 1 }, values: [1, 2, 3] },
+        { name: 'One', type: FieldType.number, values: [4, 6, 8] },
+        { name: 'Two', type: FieldType.string, values: ['a', 'b', 'c'] },
+      ],
+    });
+
+    const result = nullToValue(applyNullInsertThreshold({ frame: df, refFieldName: null }));
+
+    expect(result.fields[0].values.toArray()).toStrictEqual([1, 2, 3]);
+    expect(result.fields[1].values.toArray()).toStrictEqual([4, 6, 8]);
+    expect(result.fields[2].values.toArray()).toStrictEqual(['a', 'b', 'c']);
+  });
+});

packages/grafana-ui/src/components/GraphNG/nullToValue.ts

@@ -0,0 +1,17 @@
+import { DataFrame } from '@grafana/data';
+
+export function nullToValue(frame: DataFrame) {
+  frame.fields.forEach((f) => {
+    const noValue = +f.config?.noValue!;
+    if (!Number.isNaN(noValue)) {
+      const values = f.values.toArray();
+      for (let i = 0; i < values.length; i++) {
+        if (values[i] === null) {
+          values[i] = noValue;
+        }
+      }
+    }
+  });
+
+  return frame;
+}

packages/grafana-ui/src/components/GraphNG/utils.ts

@@ -44,7 +44,18 @@ function applySpanNullsThresholds(frame: DataFrame) {
 
 export function preparePlotFrame(frames: DataFrame[], dimFields: XYFieldMatchers, timeRange?: TimeRange | null) {
   // apply null insertions at interval
-  frames = frames.map((frame) => applyNullInsertThreshold(frame, null, timeRange?.to.valueOf()));
+  frames = frames.map((frame) => {
+    if (!frame.fields[0].state?.nullThresholdApplied) {
+      return applyNullInsertThreshold({
+        frame,
+        refFieldName: null,
+        refFieldPseudoMin: timeRange?.from.valueOf(),
+        refFieldPseudoMax: timeRange?.to.valueOf(),
+      });
+    } else {
+      return frame;
+    }
+  });
 
   let numBarSeries = 0;
 

packages/grafana-ui/src/components/Sparkline/utils.ts

@@ -14,14 +14,16 @@ export function preparePlotFrame(sparkline: FieldSparkline, config?: FieldConfig
   };
 
   return applyNullInsertThreshold({
-    refId: 'sparkline',
-    fields: [
-      sparkline.x ?? IndexVector.newField(length),
-      {
-        ...sparkline.y,
-        config: yFieldConfig,
-      },
-    ],
-    length,
+    frame: {
+      refId: 'sparkline',
+      fields: [
+        sparkline.x ?? IndexVector.newField(length),
+        {
+          ...sparkline.y,
+          config: yFieldConfig,
+        },
+      ],
+      length,
+    },
   });
 }

public/app/plugins/panel/graph/data_processor.ts

@@ -30,7 +30,7 @@ export class DataProcessor {
         continue;
       }
 
-      series = applyNullInsertThreshold(series, timeField.name);
+      series = applyNullInsertThreshold({ frame: series, refFieldName: timeField.name });
       timeField = getTimeField(series).timeField!; // use updated length
 
       for (let j = 0; j < series.fields.length; j++) {

public/app/plugins/panel/state-timeline/StateTimelinePanel.tsx

@@ -29,8 +29,8 @@ export const StateTimelinePanel: React.FC<TimelinePanelProps> = ({
   const { sync } = usePanelContext();
 
   const { frames, warn } = useMemo(
-    () => prepareTimelineFields(data?.series, options.mergeValues ?? true, theme),
-    [data, options.mergeValues, theme]
+    () => prepareTimelineFields(data?.series, options.mergeValues ?? true, timeRange, theme),
+    [data, options.mergeValues, timeRange, theme]
   );
 
   const legendItems = useMemo(

public/app/plugins/panel/state-timeline/utils.test.ts

@@ -1,4 +1,4 @@
-import { ArrayVector, createTheme, FieldType, ThresholdsMode, toDataFrame } from '@grafana/data';
+import { ArrayVector, createTheme, FieldType, ThresholdsMode, TimeRange, toDataFrame, dateTime } from '@grafana/data';
 import { LegendDisplayMode } from '@grafana/schema';
 
 import {
@@ -12,6 +12,11 @@ import {
 const theme = createTheme();
 
 describe('prepare timeline graph', () => {
+  const timeRange: TimeRange = {
+    from: dateTime(1),
+    to: dateTime(3),
+    raw: {} as any,
+  };
   it('errors with no time fields', () => {
     const frames = [
       toDataFrame({
@@ -21,7 +26,7 @@ describe('prepare timeline graph', () => {
         ],
       }),
     ];
-    const info = prepareTimelineFields(frames, true, theme);
+    const info = prepareTimelineFields(frames, true, timeRange, theme);
     expect(info.warn).toEqual('Data does not have a time field');
   });
 
@@ -34,7 +39,7 @@ describe('prepare timeline graph', () => {
         ],
       }),
     ];
-    const info = prepareTimelineFields(frames, true, theme);
+    const info = prepareTimelineFields(frames, true, timeRange, theme);
     expect(info.warn).toEqual('No graphable fields');
   });
 
@@ -47,7 +52,7 @@ describe('prepare timeline graph', () => {
         ],
       }),
     ];
-    const info = prepareTimelineFields(frames, true, theme);
+    const info = prepareTimelineFields(frames, true, timeRange, theme);
     expect(info.warn).toBeUndefined();
 
     const out = info.frames![0];

public/app/plugins/panel/state-timeline/utils.ts

@@ -21,6 +21,7 @@ import {
   Threshold,
   getFieldConfigWithMinMax,
   ThresholdsMode,
+  TimeRange,
 } from '@grafana/data';
 import { VizLegendOptions, AxisPlacement, ScaleDirection, ScaleOrientation } from '@grafana/schema';
 import {
@@ -30,6 +31,8 @@ import {
   UPlotConfigPrepFn,
   VizLegendItem,
 } from '@grafana/ui';
+import { applyNullInsertThreshold } from '@grafana/ui/src/components/GraphNG/nullInsertThreshold';
+import { nullToValue } from '@grafana/ui/src/components/GraphNG/nullToValue';
 import { PlotTooltipInterpolator } from '@grafana/ui/src/components/uPlot/types';
 
 import { preparePlotData2, getStackingGroups } from '../../../../../packages/grafana-ui/src/components/uPlot/utils';
@@ -379,6 +382,7 @@ export function mergeThresholdValues(field: Field, theme: GrafanaTheme2): Field
 export function prepareTimelineFields(
   series: DataFrame[] | undefined,
   mergeValues: boolean,
+  timeRange: TimeRange,
   theme: GrafanaTheme2
 ): { frames?: DataFrame[]; warn?: string } {
   if (!series?.length) {
@@ -386,11 +390,25 @@ export function prepareTimelineFields(
   }
   let hasTimeseries = false;
   const frames: DataFrame[] = [];
+
   for (let frame of series) {
     let isTimeseries = false;
     let changed = false;
+
+    let nulledFrame = applyNullInsertThreshold({
+      frame,
+      refFieldPseudoMin: timeRange.from.valueOf(),
+      refFieldPseudoMax: timeRange.to.valueOf(),
+    });
+
+    // Mark the field state as having a null threhold applied
+    frame.fields[0].state = {
+      ...frame.fields[0].state,
+      nullThresholdApplied: true,
+    };
+
     const fields: Field[] = [];
-    for (let field of frame.fields) {
+    for (let field of nullToValue(nulledFrame).fields) {
       switch (field.type) {
         case FieldType.time:
           isTimeseries = true;

public/app/plugins/panel/status-history/StatusHistoryPanel.tsx

@@ -26,7 +26,10 @@ export const StatusHistoryPanel: React.FC<TimelinePanelProps> = ({
 }) => {
   const theme = useTheme2();
 
-  const { frames, warn } = useMemo(() => prepareTimelineFields(data?.series, false, theme), [data, theme]);
+  const { frames, warn } = useMemo(
+    () => prepareTimelineFields(data?.series, false, timeRange, theme),
+    [data, timeRange, theme]
+  );
 
   const legendItems = useMemo(
     () => prepareTimelineLegendItems(frames, options.legend, theme),