Heatmap (new): add exemplar mapping function (#48780)

public/app/plugins/panel/heatmap-new/HeatmapPanel.tsx

@@ -42,7 +42,7 @@ export const HeatmapPanel: React.FC<HeatmapPanelProps> = ({
   let timeRangeRef = useRef<TimeRange>(timeRange);
   timeRangeRef.current = timeRange;
 
-  const info = useMemo(() => prepareHeatmapData(data.series, options, theme), [data, options, theme]);
+  const info = useMemo(() => prepareHeatmapData(data, options, theme), [data, options, theme]);
 
   const facets = useMemo(() => [null, info.heatmap?.fields.map((f) => f.values.toArray())], [info.heatmap]);
 

public/app/plugins/panel/heatmap-new/fields.test.ts

@@ -1,5 +1,6 @@
-import { createTheme } from '@grafana/data';
+import { createTheme, ArrayVector, DataFrameType, FieldType } from '@grafana/data';
 
+import { BucketLayout, getAnnotationMapping, HEATMAP_NOT_SCANLINES_ERROR } from './fields';
 import { PanelOptions } from './models.gen';
 
 const theme = createTheme();
@@ -12,3 +13,312 @@ describe('Heatmap data', () => {
     expect(options).toBeDefined();
   });
 });
+
+describe('creating a heatmap data mapping', () => {
+  describe('generates a simple data mapping with orderly data', () => {
+    const mapping = getAnnotationMapping(
+      {
+        heatmap: {
+          name: 'test',
+          meta: {
+            type: DataFrameType.HeatmapScanlines,
+          },
+          fields: [
+            {
+              name: 'xMin',
+              type: FieldType.number,
+              config: {},
+              values: new ArrayVector([1]),
+            },
+            {
+              name: 'yMin',
+              type: FieldType.number,
+              config: {},
+              values: new ArrayVector([1, 4, 7]),
+            },
+            {
+              name: 'count',
+              type: FieldType.number,
+              config: {},
+              values: new ArrayVector([3, 3, 3]),
+            },
+          ],
+          length: 3,
+        },
+        xBucketCount: 1,
+        xBucketSize: 9,
+        xLayout: BucketLayout.ge,
+        yBucketCount: 3,
+        yBucketSize: 3,
+        yLayout: BucketLayout.ge,
+      },
+      {
+        name: 'origdata',
+        fields: [
+          {
+            name: 'time',
+            type: FieldType.time,
+            config: {},
+            values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
+          },
+          {
+            name: 'value',
+            type: FieldType.number,
+            config: {},
+            values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
+          },
+        ],
+        length: 2,
+      }
+    );
+
+    it('takes good data, and delivers a working data mapping', () => {
+      expect(mapping.lookup.length).toEqual(3);
+      expect(mapping.lookup[0]).toEqual([0, 1, 2]);
+      expect(mapping.lookup[1]).toEqual([3, 4, 5]);
+      expect(mapping.lookup[2]).toEqual([6, 7, 8]);
+    });
+  });
+
+  describe('generates a data mapping with less orderly data (counts are different)', () => {
+    const heatmap = {
+      heatmap: {
+        name: 'test',
+        meta: {
+          type: DataFrameType.HeatmapScanlines,
+        },
+        fields: [
+          {
+            name: 'xMin',
+            type: FieldType.number,
+            config: {},
+            values: new ArrayVector([1]),
+          },
+          {
+            name: 'yMin',
+            type: FieldType.number,
+            config: {},
+            values: new ArrayVector([1, 4, 7]),
+          },
+          {
+            name: 'count',
+            type: FieldType.number,
+            config: {},
+            values: new ArrayVector([2, 1, 6]),
+          },
+        ],
+        length: 3,
+      },
+      xBucketCount: 1,
+      xBucketSize: 9,
+      xLayout: BucketLayout.ge,
+      yBucketCount: 3,
+      yBucketSize: 3,
+      yLayout: BucketLayout.ge,
+    };
+    const rawData = {
+      name: 'origdata',
+      fields: [
+        {
+          name: 'time',
+          type: FieldType.time,
+          config: {},
+          values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
+        },
+        {
+          name: 'value',
+          type: FieldType.number,
+          config: {},
+          values: new ArrayVector([7, 1, 8, 7, 2, 7, 8, 8, 4]),
+        },
+      ],
+      length: 2,
+    };
+
+    it("Puts data into the proper buckets, when we don't care about the count", () => {
+      // In this case, we are just finding proper values, but don't care if a values
+      // exists in the bucket in the original data or not. Therefore, we should see
+      // a value mapped into the second mapping bucket containing the value '8'.
+      const mapping = getAnnotationMapping(heatmap, rawData);
+      expect(mapping.lookup.length).toEqual(3);
+      expect(mapping.lookup[0]).toEqual([1, 4]);
+      expect(mapping.lookup[1]).toEqual([8]);
+      expect(mapping.lookup[2]).toEqual([0, 2, 3, 5, 6, 7]);
+    });
+  });
+
+  describe('Handles a larger data set that will not fill all buckets', () => {
+    const mapping = getAnnotationMapping(
+      {
+        heatmap: {
+          name: 'test',
+          meta: {
+            type: DataFrameType.HeatmapScanlines,
+          },
+          fields: [
+            {
+              name: 'xMin',
+              type: FieldType.number,
+              config: {},
+              values: new ArrayVector([1, 1, 1, 4, 4, 4, 7, 7, 7]),
+            },
+            {
+              name: 'yMin',
+              type: FieldType.number,
+              config: {},
+              values: new ArrayVector([1, 4, 7, 1, 4, 7, 1, 4, 7]),
+            },
+            {
+              name: 'count',
+              type: FieldType.number,
+              config: {},
+              values: new ArrayVector([0, 0, 2, 0, 0, 3, 0, 2, 0]),
+            },
+          ],
+          length: 3,
+        },
+        xBucketCount: 3,
+        xBucketSize: 3,
+        xLayout: BucketLayout.ge,
+        yBucketCount: 3,
+        yBucketSize: 3,
+        yLayout: BucketLayout.ge,
+      },
+      {
+        name: 'origdata',
+        fields: [
+          {
+            name: 'time',
+            type: FieldType.time,
+            config: {},
+            values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
+          },
+          {
+            name: 'value',
+            type: FieldType.number,
+            config: {},
+            values: new ArrayVector([8, 0, 8, 7, 7, 8, 6, 10, 6]),
+          },
+        ],
+        length: 2,
+      }
+    );
+
+    it('Creates the data mapping correctly', () => {
+      expect(mapping.lookup.length).toEqual(9);
+      expect(mapping).toEqual({
+        lookup: [null, null, [0, 2], null, null, [3, 4, 5], null, [6, 8], null],
+        low: [1],
+        high: [7],
+      });
+    });
+
+    it('filters out minimum and maximum values', () => {
+      expect(mapping.lookup.flat()).not.toContainEqual(1);
+      expect(mapping.lookup.flat()).not.toContainEqual(10);
+    });
+  });
+
+  describe('Error scenarios', () => {
+    const heatmap = {
+      heatmap: {
+        name: 'test',
+        meta: {
+          type: DataFrameType.HeatmapBuckets,
+        },
+        fields: [
+          {
+            name: 'xMin',
+            type: FieldType.number,
+            config: {},
+            values: new ArrayVector([1]),
+          },
+          {
+            name: 'yMin',
+            type: FieldType.number,
+            config: {},
+            values: new ArrayVector([1, 4, 7]),
+          },
+          {
+            name: 'count',
+            type: FieldType.number,
+            config: {},
+            values: new ArrayVector([2, 0, 6]),
+          },
+        ],
+        length: 3,
+      },
+      xBucketCount: 1,
+      xBucketSize: 9,
+      xLayout: BucketLayout.ge,
+      yBucketCount: 3,
+      yBucketSize: 3,
+      yLayout: BucketLayout.ge,
+    };
+    const rawData = {
+      name: 'origdata',
+      fields: [
+        {
+          name: 'time',
+          type: FieldType.time,
+          config: {},
+          values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
+        },
+        {
+          name: 'value',
+          type: FieldType.number,
+          config: {},
+          values: new ArrayVector([7, 1, 8, 7, 2, 7, 8, 8, 4]),
+        },
+      ],
+      length: 2,
+    };
+
+    it('Will not process heatmap buckets', () => {
+      expect(() =>
+        getAnnotationMapping(
+          {
+            ...heatmap,
+            heatmap: {
+              ...heatmap.heatmap,
+              meta: {
+                type: DataFrameType.HeatmapBuckets,
+              },
+            },
+          },
+          rawData
+        )
+      ).toThrow(HEATMAP_NOT_SCANLINES_ERROR);
+
+      expect(() =>
+        getAnnotationMapping(
+          {
+            ...heatmap,
+            heatmap: {
+              ...heatmap.heatmap,
+              meta: {
+                type: DataFrameType.TimeSeriesWide,
+              },
+            },
+          },
+          rawData
+        )
+      ).toThrow(HEATMAP_NOT_SCANLINES_ERROR);
+
+      expect(() =>
+        getAnnotationMapping(
+          {
+            ...heatmap,
+            heatmap: {
+              ...heatmap.heatmap,
+              meta: {
+                type: undefined,
+              },
+            },
+          },
+          rawData
+        )
+      ).toThrow(HEATMAP_NOT_SCANLINES_ERROR);
+    });
+  });
+});

public/app/plugins/panel/heatmap-new/fields.ts

@@ -1,12 +1,16 @@
 import {
   DataFrame,
   DataFrameType,
+  Field,
   FieldType,
   formattedValueToString,
   getDisplayProcessor,
   getFieldDisplayName,
   getValueFormat,
   GrafanaTheme2,
+  incrRoundDn,
+  incrRoundUp,
+  PanelData,
 } from '@grafana/data';
 import { calculateHeatmapFromData, bucketsToScanlines } from 'app/features/transformers/calculateHeatmap/heatmap';
 
@@ -17,9 +21,19 @@ export const enum BucketLayout {
   ge = 'ge',
 }
 
+export interface HeatmapDataMapping {
+  lookup: Array<number[] | null>;
+  high: number[]; // index of values bigger than the max Y
+  low: number[]; // index of values less than the min Y
+}
+
+export const HEATMAP_NOT_SCANLINES_ERROR = 'A calculated heatmap was expected, but not found';
+
 export interface HeatmapData {
   // List of heatmap frames
   heatmap?: DataFrame;
+  annotations?: DataFrame;
+  annotationMappings?: HeatmapDataMapping;
 
   yAxisValues?: Array<number | string | null>;
 
@@ -39,25 +53,25 @@ export interface HeatmapData {
   warning?: string;
 }
 
-export function prepareHeatmapData(
+export function prepareHeatmapData(data: PanelData, options: PanelOptions, theme: GrafanaTheme2): HeatmapData {
-  frames: DataFrame[] | undefined,
+  const frames = data.series;
-  options: PanelOptions,
-  theme: GrafanaTheme2
-): HeatmapData {
   if (!frames?.length) {
     return {};
   }
 
   const { source } = options;
+
+  const annotations = data.annotations?.[0]; // TODO: Maybe join on time with frames
+
   if (source === HeatmapSourceMode.Calculate) {
     // TODO, check for error etc
-    return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), theme);
+    return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), annotations, theme);
   }
 
   // Find a well defined heatmap
   let scanlinesHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapScanlines);
   if (scanlinesHeatmap) {
-    return getHeatmapData(scanlinesHeatmap, theme);
+    return getHeatmapData(scanlinesHeatmap, annotations, theme);
   }
 
   let bucketsHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapBuckets);
@@ -66,19 +80,79 @@ export function prepareHeatmapData(
       yAxisValues: frames[0].fields.flatMap((field) =>
         field.type === FieldType.number ? getFieldDisplayName(field) : []
       ),
-      ...getHeatmapData(bucketsToScanlines(bucketsHeatmap), theme),
+      ...getHeatmapData(bucketsToScanlines(bucketsHeatmap), annotations, theme),
     };
   }
 
   if (source === HeatmapSourceMode.Data) {
-    return getHeatmapData(bucketsToScanlines(frames[0]), theme);
+    return getHeatmapData(bucketsToScanlines(frames[0]), annotations, theme);
   }
 
   // TODO, check for error etc
-  return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), theme);
+  return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), annotations, theme);
 }
 
-const getHeatmapData = (frame: DataFrame, theme: GrafanaTheme2): HeatmapData => {
+const getHeatmapFields = (dataFrame: DataFrame): Array<Field | undefined> => {
+  const xField: Field | undefined = dataFrame.fields.find((f) => f.name === 'xMin');
+  const yField: Field | undefined = dataFrame.fields.find((f) => f.name === 'yMin');
+  const countField: Field | undefined = dataFrame.fields.find((f) => f.name === 'count');
+
+  return [xField, yField, countField];
+};
+
+export const getAnnotationMapping = (heatmapData: HeatmapData, rawData: DataFrame): HeatmapDataMapping => {
+  if (heatmapData.heatmap?.meta?.type !== DataFrameType.HeatmapScanlines) {
+    throw HEATMAP_NOT_SCANLINES_ERROR;
+  }
+
+  const [fxs, fys] = getHeatmapFields(heatmapData.heatmap!);
+
+  if (!fxs || !fys) {
+    throw HEATMAP_NOT_SCANLINES_ERROR;
+  }
+
+  const mapping: HeatmapDataMapping = {
+    lookup: new Array(heatmapData.xBucketCount! * heatmapData.yBucketCount!).fill(null),
+    high: [],
+    low: [],
+  };
+
+  const xos: number[] | undefined = rawData.fields.find((f: Field) => f.type === 'time')?.values.toArray();
+  const yos: number[] | undefined = rawData.fields.find((f: Field) => f.type === 'number')?.values.toArray();
+
+  if (!xos || !yos) {
+    return mapping;
+  }
+
+  const xsmin = fxs.values.get(0);
+  const ysmin = fys.values.get(0);
+  const xsmax = fxs.values.get(fxs.values.length - 1) + heatmapData.xBucketSize!;
+  const ysmax = fys.values.get(fys.values.length - 1) + heatmapData.yBucketSize!;
+  xos.forEach((xo: number, i: number) => {
+    const yo = yos[i];
+    const xBucketIdx = Math.floor(incrRoundDn(incrRoundUp((xo - xsmin) / heatmapData.xBucketSize!, 1e-7), 1e-7));
+    const yBucketIdx = Math.floor(incrRoundDn(incrRoundUp((yo - ysmin) / heatmapData.yBucketSize!, 1e-7), 1e-7));
+
+    if (xo < xsmin || yo < ysmin) {
+      mapping.low.push(i);
+      return;
+    }
+
+    if (xo >= xsmax || yo >= ysmax) {
+      mapping.high.push(i);
+      return;
+    }
+
+    const index = xBucketIdx * heatmapData.yBucketCount! + yBucketIdx;
+    if (mapping.lookup[index] === null) {
+      mapping.lookup[index] = [];
+    }
+    mapping.lookup[index]?.push(i);
+  });
+  return mapping;
+};
+
+const getHeatmapData = (frame: DataFrame, annotations: DataFrame | undefined, theme: GrafanaTheme2): HeatmapData => {
   if (frame.meta?.type !== DataFrameType.HeatmapScanlines) {
     return {
       warning: 'Expected heatmap scanlines format',
@@ -114,8 +188,9 @@ const getHeatmapData = (frame: DataFrame, theme: GrafanaTheme2): HeatmapData =>
 
   // The "count" field
   const disp = frame.fields[2].display ?? getValueFormat('short');
-  return {
+  const data: HeatmapData = {
     heatmap: frame,
+    annotations,
     xBucketSize: xBinIncr,
     yBucketSize: yBinIncr,
     xBucketCount: xBinQty,
@@ -127,4 +202,10 @@ const getHeatmapData = (frame: DataFrame, theme: GrafanaTheme2): HeatmapData =>
 
     display: (v) => formattedValueToString(disp(v)),
   };
+
+  if (annotations) {
+    data.annotationMappings = getAnnotationMapping(data, annotations);
+  }
+
+  return data;
 };

public/app/plugins/panel/heatmap-new/suggestions.ts

@@ -18,7 +18,7 @@ export class HeatmapSuggestionsSupplier {
       return;
     }
 
-    const info = prepareHeatmapData(builder.data.series, defaultPanelOptions, config.theme2);
+    const info = prepareHeatmapData(builder.data, defaultPanelOptions, config.theme2);
     if (!info || info.warning) {
       return;
     }

public/app/plugins/panel/heatmap-new/utils.test.ts

@@ -0,0 +1,5 @@
+describe('a test', () => {
+  it('has to have at least one test', () => {
+    expect(true).toBeTruthy();
+  });
+});