Time Series Panel: Add Null Filling and "No Value" Support (#50907)

* Use nullInsertThreshold and nullToValue in time series

* Allow for undefined timeRange to support certain candlestick uses of prepareGraphableFields

* Make sure null to value doesn't modify initial data

* Do a shallow values copy and avoid Array.push()

* Clean up null to value transformation.

* Add basic tests

* Remove redunant null threshold application flagging

* set nullThresholdApplied flag even when no null inserts were done

* Include nullThresholdApplied in test snapshot

Co-authored-by: Leon Sorokin <leeoniya@gmail.com>

packages/grafana-ui/src/components/GraphNG/nullInsertThreshold.ts

@@ -38,6 +38,11 @@ export function applyNullInsertThreshold(opts: NullInsertOptions): DataFrame {
     return frame;
   }
 
+  refField.state = {
+    ...refField.state,
+    nullThresholdApplied: true,
+  };
+
   const thresholds = frame.fields.map((field) => field.config.custom?.insertNulls ?? refField.config.interval ?? null);
 
   const uniqueThresholds = new Set<number>(thresholds);
@@ -76,21 +81,10 @@ export function applyNullInsertThreshold(opts: NullInsertOptions): DataFrame {
     return {
       ...frame,
       length: filledFieldValues[0].length,
-      fields: frame.fields.map((field, i) => {
+      fields: frame.fields.map((field, i) => ({
-        let f = {
+        ...field,
-          ...field,
+        values: new ArrayVector(filledFieldValues[i]),
-          values: new ArrayVector(filledFieldValues[i]),
+      })),
-        };
-
-        if (i === 0) {
-          f.state = {
-            ...field.state,
-            nullThresholdApplied: true,
-          };
-        }
-
-        return f;
-      }),
     };
   }
 

packages/grafana-ui/src/components/GraphNG/nullToValue.ts

@@ -1,17 +1,27 @@
-import { DataFrame } from '@grafana/data';
+import { ArrayVector, DataFrame } from '@grafana/data';
 
 export function nullToValue(frame: DataFrame) {
-  frame.fields.forEach((f) => {
+  return {
-    const noValue = +f.config?.noValue!;
+    ...frame,
-    if (!Number.isNaN(noValue)) {
+    fields: frame.fields.map((field) => {
-      const values = f.values.toArray();
+      const noValue = +field.config?.noValue!;
-      for (let i = 0; i < values.length; i++) {
-        if (values[i] === null) {
-          values[i] = noValue;
-        }
-      }
-    }
-  });
 
-  return frame;
+      if (!Number.isNaN(noValue)) {
+        const transformedVals = field.values.toArray().slice();
+
+        for (let i = 0; i < transformedVals.length; i++) {
+          if (transformedVals[i] === null) {
+            transformedVals[i] = noValue;
+          }
+        }
+
+        return {
+          ...field,
+          values: new ArrayVector(transformedVals),
+        };
+      } else {
+        return field;
+      }
+    }),
+  };
 }

packages/grafana-ui/src/components/GraphNG/utils.test.ts

@@ -328,6 +328,7 @@ describe('GraphNG utils', () => {
             "config": Object {},
             "name": "time",
             "state": Object {
+              "nullThresholdApplied": true,
               "origin": Object {
                 "fieldIndex": 0,
                 "frameIndex": 0,

public/app/plugins/panel/candlestick/CandlestickPanel.tsx

@@ -46,7 +46,9 @@ export const CandlestickPanel: React.FC<CandlestickPanelProps> = ({
 
   const theme = useTheme2();
 
-  const info = useMemo(() => prepareCandlestickFields(data?.series, options, theme), [data, options, theme]);
+  const info = useMemo(() => {
+    return prepareCandlestickFields(data?.series, options, theme, timeRange);
+  }, [data, options, theme, timeRange]);
 
   const { renderers, tweakScale, tweakAxis } = useMemo(() => {
     let tweakScale = (opts: ScaleProps, forField: Field) => opts;

public/app/plugins/panel/candlestick/fields.ts

@@ -6,6 +6,7 @@ import {
   getFieldDisplayName,
   GrafanaTheme2,
   outerJoinDataFrames,
+  TimeRange,
 } from '@grafana/data';
 import { maybeSortFrame } from '@grafana/data/src/transformations/transformers/joinDataFrames';
 import { findField } from 'app/features/dimensions';
@@ -97,7 +98,8 @@ function findFieldOrAuto(frame: DataFrame, info: FieldPickerInfo, options: Candl
 export function prepareCandlestickFields(
   series: DataFrame[] | undefined,
   options: CandlestickOptions,
-  theme: GrafanaTheme2
+  theme: GrafanaTheme2,
+  timeRange?: TimeRange
 ): CandlestickData | null {
   if (!series?.length) {
     return null;
@@ -119,7 +121,7 @@ export function prepareCandlestickFields(
   const data: CandlestickData = { aligned, frame: aligned, names: {} };
 
   // Apply same filter as everythign else in timeseries
-  const timeSeriesFrames = prepareGraphableFields([aligned], theme);
+  const timeSeriesFrames = prepareGraphableFields([aligned], theme, timeRange);
   if (!timeSeriesFrames) {
     return null;
   }

public/app/plugins/panel/timeseries/TimeSeriesPanel.tsx

@@ -36,7 +36,7 @@ export const TimeSeriesPanel: React.FC<TimeSeriesPanelProps> = ({
     return getFieldLinksForExplore({ field, rowIndex, splitOpenFn: onSplitOpen, range: timeRange });
   };
 
-  const frames = useMemo(() => prepareGraphableFields(data.series, config.theme2), [data]);
+  const frames = useMemo(() => prepareGraphableFields(data.series, config.theme2, timeRange), [data, timeRange]);
 
   if (!frames) {
     return (

public/app/plugins/panel/timeseries/utils.test.ts

@@ -79,4 +79,48 @@ describe('prepare timeseries graph', () => {
       ]
     `);
   });
+
+  it('will insert nulls given an interval value', () => {
+    const df = new MutableDataFrame({
+      fields: [
+        { name: 'time', type: FieldType.time, config: { interval: 1 }, values: [1, 3, 6] },
+        { name: 'a', values: [1, 2, 3] },
+      ],
+    });
+    const frames = prepareGraphableFields([df], createTheme());
+
+    const field = frames![0].fields.find((f) => f.name === 'a');
+    expect(field!.values.toArray()).toMatchInlineSnapshot(`
+      Array [
+        1,
+        null,
+        2,
+        null,
+        null,
+        3,
+      ]
+    `);
+  });
+
+  it('will insert and convert nulls to a configure "no value" value', () => {
+    const df = new MutableDataFrame({
+      fields: [
+        { name: 'time', type: FieldType.time, config: { interval: 1 }, values: [1, 3, 6] },
+        { name: 'a', config: { noValue: '20' }, values: [1, 2, 3] },
+      ],
+    });
+    const frames = prepareGraphableFields([df], createTheme());
+
+    const field = frames![0].fields.find((f) => f.name === 'a');
+    expect(field!.values.toArray()).toMatchInlineSnapshot(`
+      Array [
+        1,
+        20,
+        2,
+        20,
+        20,
+        3,
+      ]
+    `);
+  });
 });

public/app/plugins/panel/timeseries/utils.ts

@@ -6,13 +6,20 @@ import {
   getDisplayProcessor,
   GrafanaTheme2,
   isBooleanUnit,
+  TimeRange,
 } from '@grafana/data';
 import { GraphFieldConfig, LineInterpolation } from '@grafana/schema';
+import { applyNullInsertThreshold } from '@grafana/ui/src/components/GraphNG/nullInsertThreshold';
+import { nullToValue } from '@grafana/ui/src/components/GraphNG/nullToValue';
 
 /**
  * Returns null if there are no graphable fields
  */
-export function prepareGraphableFields(series: DataFrame[], theme: GrafanaTheme2): DataFrame[] | null {
+export function prepareGraphableFields(
+  series: DataFrame[],
+  theme: GrafanaTheme2,
+  timeRange?: TimeRange
+): DataFrame[] | null {
   if (!series?.length) {
     return null;
   }
@@ -27,7 +34,13 @@ export function prepareGraphableFields(series: DataFrame[], theme: GrafanaTheme2
     let hasTimeField = false;
     let hasValueField = false;
 
-    for (const field of frame.fields) {
+    let nulledFrame = applyNullInsertThreshold({
+      frame,
+      refFieldPseudoMin: timeRange?.from.valueOf(),
+      refFieldPseudoMax: timeRange?.to.valueOf(),
+    });
+
+    for (const field of nullToValue(nulledFrame).fields) {
       switch (field.type) {
         case FieldType.time:
           hasTimeField = true;