HeatmapNG: render exemplars (#49287)

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

public/app/features/transformers/calculateHeatmap/heatmap.ts

@@ -97,7 +97,8 @@ export function bucketsToScanlines(frame: DataFrame): DataFrame {
         config: xField.config,
       },
       {
-        name: 'yMax',
+        // this name determines whether cells are drawn above, below, or centered on the values
+        name: yField.labels?.le != null ? 'yMax' : 'y',
         type: FieldType.number,
         values: new ArrayVector(ys),
         config: yField.config,

public/app/plugins/panel/heatmap-new/HeatmapHoverView.tsx

@@ -1,14 +1,6 @@
 import React, { useEffect, useRef } from 'react';
 
-import {
+import { DataFrameType, Field, FieldType, formattedValueToString, getFieldDisplayName, LinkModel } from '@grafana/data';
-  DataFrameType,
-  DataFrameView,
-  Field,
-  FieldType,
-  formattedValueToString,
-  getFieldDisplayName,
-  LinkModel,
-} from '@grafana/data';
 import { LinkButton, VerticalGroup } from '@grafana/ui';
 import { getDashboardSrv } from 'app/features/dashboard/services/DashboardSrv';
 
@@ -23,7 +15,15 @@ type Props = {
   showHistogram?: boolean;
 };
 
-export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
+export const HeatmapHoverView = (props: Props) => {
+  if (props.hover.seriesIdx === 2) {
+    return <DataHoverView data={props.data.exemplars} rowIndex={props.hover.dataIdx} />;
+  }
+  return <HeatmapHoverCell {...props} />;
+};
+
+const HeatmapHoverCell = ({ data, hover, showHistogram }: Props) => {
+  const index = hover.dataIdx;
   const xField = data.heatmap?.fields[0];
   const yField = data.heatmap?.fields[1];
   const countField = data.heatmap?.fields[2];
@@ -60,7 +60,7 @@ export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
     };
   }
 
-  const yValueIdx = hover.index % data.yBucketCount! ?? 0;
+  const yValueIdx = index % data.yBucketCount! ?? 0;
 
   const yMinIdx = data.yLayout === BucketLayout.le ? yValueIdx - 1 : yValueIdx;
   const yMaxIdx = data.yLayout === BucketLayout.le ? yValueIdx : yValueIdx + 1;
@@ -68,10 +68,10 @@ export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
   const yBucketMin = yDispSrc?.[yMinIdx];
   const yBucketMax = yDispSrc?.[yMaxIdx];
 
-  const xBucketMin = xVals?.[hover.index];
+  const xBucketMin = xVals?.[index];
   const xBucketMax = xBucketMin + data.xBucketSize;
 
-  const count = countVals?.[hover.index];
+  const count = countVals?.[index];
 
   const visibleFields = data.heatmap?.fields.filter((f) => !Boolean(f.config.custom?.hideFrom?.tooltip));
   const links: Array<LinkModel<Field>> = [];
@@ -80,10 +80,10 @@ export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
   for (const field of visibleFields ?? []) {
     // TODO: Currently always undefined? (getLinks)
     if (field.getLinks) {
-      const v = field.values.get(hover.index);
+      const v = field.values.get(index);
       const disp = field.display ? field.display(v) : { text: `${v}`, numeric: +v };
 
-      field.getLinks({ calculatedValue: disp, valueRowIndex: hover.index }).forEach((link) => {
+      field.getLinks({ calculatedValue: disp, valueRowIndex: index }).forEach((link) => {
         const key = `${link.title}/${link.href}`;
         if (!linkLookup.has(key)) {
           links.push(link);
@@ -106,9 +106,9 @@ export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
         let histCtx = can.current?.getContext('2d');
 
         if (histCtx && xVals && yVals && countVals) {
-          let fromIdx = hover.index;
+          let fromIdx = index;
 
-          while (xVals[fromIdx--] === xVals[hover.index]) {}
+          while (xVals[fromIdx--] === xVals[index]) {}
 
           fromIdx++;
 
@@ -135,7 +135,7 @@ export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
               let pctY = c / maxCount;
               let pctX = j / (data.yBucketCount! + 1);
 
-              let p = i === hover.index ? pHov : pRest;
+              let p = i === index ? pHov : pRest;
 
               p.rect(
                 Math.round(histCanWidth * pctX),
@@ -160,37 +160,13 @@ export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
       }
     },
     // eslint-disable-next-line react-hooks/exhaustive-deps
-    [hover.index]
+    [index]
   );
 
-  const renderExemplars = () => {
-    const exemplarIndex = data.exemplarsMappings?.lookup; //?.[hover.index];
-    if (!exemplarIndex || !data.exemplars) {
-      return null;
-    }
-
-    const ids = exemplarIndex[hover.index];
-    if (ids) {
-      const view = new DataFrameView(data.exemplars);
-      return (
-        <ul>
-          {ids.map((id) => (
-            <li key={id}>
-              <pre>{JSON.stringify(view.get(id), null, 2)}</pre>
-            </li>
-          ))}
-        </ul>
-      );
-    }
-
-    // should not show anything... but for debugging
-    return <div>EXEMPLARS: {JSON.stringify(exemplarIndex)}</div>;
-  };
-
   if (data.heatmap?.meta?.type === DataFrameType.HeatmapSparse) {
     return (
       <div>
-        <DataHoverView data={data.heatmap} rowIndex={hover.index} />
+        <DataHoverView data={data.heatmap} rowIndex={index} />
       </div>
     );
   }
@@ -210,14 +186,17 @@ export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
         />
       )}
       <div>
+        {data.yLayout === BucketLayout.unknown ? (
+          <div>{yDisp(yBucketMin)}</div>
+        ) : (
           <div>
             Bucket: {yDisp(yBucketMin)} - {yDisp(yBucketMax)}
           </div>
+        )}
         <div>
           {getFieldDisplayName(countField!, data.heatmap)}: {count}
         </div>
       </div>
-      {renderExemplars()}
       {links.length > 0 && (
         <VerticalGroup>
           {links.map((link, i) => (

public/app/plugins/panel/heatmap-new/HeatmapPanel.tsx

@@ -42,9 +42,28 @@ export const HeatmapPanel: React.FC<HeatmapPanelProps> = ({
     }
   }, [data, options, theme]);
 
-  const facets = useMemo(() => [null, info.heatmap?.fields.map((f) => f.values.toArray())], [info.heatmap]);
+  const facets = useMemo(() => {
+    let exemplarsXFacet: number[] = []; // "Time" field
+    let exemplarsyFacet: number[] = [];
 
-  //console.log(facets);
+    if (info.exemplars && info.matchByLabel) {
+      exemplarsXFacet = info.exemplars?.fields[0].values.toArray();
+
+      // ordinal/labeled heatmap-buckets?
+      const hasLabeledY = info.yLabelValues != null;
+
+      if (hasLabeledY) {
+        let matchExemplarsBy = info.exemplars?.fields
+          .find((field) => field.name === info.matchByLabel)!
+          .values.toArray();
+        exemplarsyFacet = matchExemplarsBy.map((label) => info.yLabelValues?.indexOf(label)) as number[];
+      } else {
+        exemplarsyFacet = info.exemplars?.fields[1].values.toArray() as number[]; // "Value" field
+      }
+    }
+
+    return [null, info.heatmap?.fields.map((f) => f.values.toArray()), [exemplarsXFacet, exemplarsyFacet]];
+  }, [info.heatmap, info.exemplars, info.yLabelValues, info.matchByLabel]);
 
   const palette = useMemo(() => quantizeScheme(options.color, theme), [options.color, theme]);
 
@@ -95,6 +114,7 @@ export const HeatmapPanel: React.FC<HeatmapPanelProps> = ({
       palette,
       cellGap: options.cellGap,
       hideThreshold: options.hideThreshold,
+      exemplarColor: options.exemplars?.color ?? 'rgba(255,0,255,0.7)',
     });
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [options, data.structureRev]);
@@ -111,8 +131,9 @@ export const HeatmapPanel: React.FC<HeatmapPanelProps> = ({
     const { min, max } = reduceField({ field: countField, reducers: [ReducerID.min, ReducerID.max] });
 
     let hoverValue: number | undefined = undefined;
-    if (hover && info.heatmap.fields) {
+    // seriesIdx: 1 is heatmap layer; 2 is exemplar layer
-      hoverValue = countField.values.get(hover.index);
+    if (hover && info.heatmap.fields && hover.seriesIdx === 1) {
+      hoverValue = countField.values.get(hover.dataIdx);
     }
 
     return (

public/app/plugins/panel/heatmap-new/fields.test.ts

@@ -1,6 +1,5 @@
-import { createTheme, ArrayVector, DataFrameType, FieldType } from '@grafana/data';
+import { createTheme } from '@grafana/data';
 
-import { BucketLayout, getExemplarsMapping, HEATMAP_NOT_SCANLINES_ERROR } from './fields';
 import { PanelOptions } from './models.gen';
 
 const theme = createTheme();
@@ -13,312 +12,3 @@ describe('Heatmap data', () => {
     expect(options).toBeDefined();
   });
 });
-
-describe('creating a heatmap data mapping', () => {
-  describe('generates a simple data mapping with orderly data', () => {
-    const mapping = getExemplarsMapping(
-      {
-        heatmap: {
-          name: 'test',
-          meta: {
-            type: DataFrameType.HeatmapScanlines,
-          },
-          fields: [
-            {
-              name: 'xMin',
-              type: FieldType.number,
-              config: {},
-              values: new ArrayVector([1]),
-            },
-            {
-              name: 'yMin',
-              type: FieldType.number,
-              config: {},
-              values: new ArrayVector([1, 4, 7]),
-            },
-            {
-              name: 'count',
-              type: FieldType.number,
-              config: {},
-              values: new ArrayVector([3, 3, 3]),
-            },
-          ],
-          length: 3,
-        },
-        xBucketCount: 1,
-        xBucketSize: 9,
-        xLayout: BucketLayout.ge,
-        yBucketCount: 3,
-        yBucketSize: 3,
-        yLayout: BucketLayout.ge,
-      },
-      {
-        name: 'origdata',
-        fields: [
-          {
-            name: 'time',
-            type: FieldType.time,
-            config: {},
-            values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
-          },
-          {
-            name: 'value',
-            type: FieldType.number,
-            config: {},
-            values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
-          },
-        ],
-        length: 2,
-      }
-    );
-
-    it('takes good data, and delivers a working data mapping', () => {
-      expect(mapping.lookup.length).toEqual(3);
-      expect(mapping.lookup[0]).toEqual([0, 1, 2]);
-      expect(mapping.lookup[1]).toEqual([3, 4, 5]);
-      expect(mapping.lookup[2]).toEqual([6, 7, 8]);
-    });
-  });
-
-  describe('generates a data mapping with less orderly data (counts are different)', () => {
-    const heatmap = {
-      heatmap: {
-        name: 'test',
-        meta: {
-          type: DataFrameType.HeatmapScanlines,
-        },
-        fields: [
-          {
-            name: 'xMin',
-            type: FieldType.number,
-            config: {},
-            values: new ArrayVector([1]),
-          },
-          {
-            name: 'yMin',
-            type: FieldType.number,
-            config: {},
-            values: new ArrayVector([1, 4, 7]),
-          },
-          {
-            name: 'count',
-            type: FieldType.number,
-            config: {},
-            values: new ArrayVector([2, 1, 6]),
-          },
-        ],
-        length: 3,
-      },
-      xBucketCount: 1,
-      xBucketSize: 9,
-      xLayout: BucketLayout.ge,
-      yBucketCount: 3,
-      yBucketSize: 3,
-      yLayout: BucketLayout.ge,
-    };
-    const rawData = {
-      name: 'origdata',
-      fields: [
-        {
-          name: 'time',
-          type: FieldType.time,
-          config: {},
-          values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
-        },
-        {
-          name: 'value',
-          type: FieldType.number,
-          config: {},
-          values: new ArrayVector([7, 1, 8, 7, 2, 7, 8, 8, 4]),
-        },
-      ],
-      length: 2,
-    };
-
-    it("Puts data into the proper buckets, when we don't care about the count", () => {
-      // In this case, we are just finding proper values, but don't care if a values
-      // exists in the bucket in the original data or not. Therefore, we should see
-      // a value mapped into the second mapping bucket containing the value '8'.
-      const mapping = getExemplarsMapping(heatmap, rawData);
-      expect(mapping.lookup.length).toEqual(3);
-      expect(mapping.lookup[0]).toEqual([1, 4]);
-      expect(mapping.lookup[1]).toEqual([8]);
-      expect(mapping.lookup[2]).toEqual([0, 2, 3, 5, 6, 7]);
-    });
-  });
-
-  describe('Handles a larger data set that will not fill all buckets', () => {
-    const mapping = getExemplarsMapping(
-      {
-        heatmap: {
-          name: 'test',
-          meta: {
-            type: DataFrameType.HeatmapScanlines,
-          },
-          fields: [
-            {
-              name: 'xMin',
-              type: FieldType.number,
-              config: {},
-              values: new ArrayVector([1, 1, 1, 4, 4, 4, 7, 7, 7]),
-            },
-            {
-              name: 'yMin',
-              type: FieldType.number,
-              config: {},
-              values: new ArrayVector([1, 4, 7, 1, 4, 7, 1, 4, 7]),
-            },
-            {
-              name: 'count',
-              type: FieldType.number,
-              config: {},
-              values: new ArrayVector([0, 0, 2, 0, 0, 3, 0, 2, 0]),
-            },
-          ],
-          length: 3,
-        },
-        xBucketCount: 3,
-        xBucketSize: 3,
-        xLayout: BucketLayout.ge,
-        yBucketCount: 3,
-        yBucketSize: 3,
-        yLayout: BucketLayout.ge,
-      },
-      {
-        name: 'origdata',
-        fields: [
-          {
-            name: 'time',
-            type: FieldType.time,
-            config: {},
-            values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
-          },
-          {
-            name: 'value',
-            type: FieldType.number,
-            config: {},
-            values: new ArrayVector([8, 0, 8, 7, 7, 8, 6, 10, 6]),
-          },
-        ],
-        length: 2,
-      }
-    );
-
-    it('Creates the data mapping correctly', () => {
-      expect(mapping.lookup.length).toEqual(9);
-      expect(mapping).toEqual({
-        lookup: [null, null, [0, 2], null, null, [3, 4, 5], null, [6, 8], null],
-        low: [1],
-        high: [7],
-      });
-    });
-
-    it('filters out minimum and maximum values', () => {
-      expect(mapping.lookup.flat()).not.toContainEqual(1);
-      expect(mapping.lookup.flat()).not.toContainEqual(10);
-    });
-  });
-
-  describe('Error scenarios', () => {
-    const heatmap = {
-      heatmap: {
-        name: 'test',
-        meta: {
-          type: DataFrameType.HeatmapBuckets,
-        },
-        fields: [
-          {
-            name: 'xMin',
-            type: FieldType.number,
-            config: {},
-            values: new ArrayVector([1]),
-          },
-          {
-            name: 'yMin',
-            type: FieldType.number,
-            config: {},
-            values: new ArrayVector([1, 4, 7]),
-          },
-          {
-            name: 'count',
-            type: FieldType.number,
-            config: {},
-            values: new ArrayVector([2, 0, 6]),
-          },
-        ],
-        length: 3,
-      },
-      xBucketCount: 1,
-      xBucketSize: 9,
-      xLayout: BucketLayout.ge,
-      yBucketCount: 3,
-      yBucketSize: 3,
-      yLayout: BucketLayout.ge,
-    };
-    const rawData = {
-      name: 'origdata',
-      fields: [
-        {
-          name: 'time',
-          type: FieldType.time,
-          config: {},
-          values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]),
-        },
-        {
-          name: 'value',
-          type: FieldType.number,
-          config: {},
-          values: new ArrayVector([7, 1, 8, 7, 2, 7, 8, 8, 4]),
-        },
-      ],
-      length: 2,
-    };
-
-    it('Will not process heatmap buckets', () => {
-      expect(() =>
-        getExemplarsMapping(
-          {
-            ...heatmap,
-            heatmap: {
-              ...heatmap.heatmap,
-              meta: {
-                type: DataFrameType.HeatmapBuckets,
-              },
-            },
-          },
-          rawData
-        )
-      ).toThrow(HEATMAP_NOT_SCANLINES_ERROR);
-
-      expect(() =>
-        getExemplarsMapping(
-          {
-            ...heatmap,
-            heatmap: {
-              ...heatmap.heatmap,
-              meta: {
-                type: DataFrameType.TimeSeriesWide,
-              },
-            },
-          },
-          rawData
-        )
-      ).toThrow(HEATMAP_NOT_SCANLINES_ERROR);
-
-      expect(() =>
-        getExemplarsMapping(
-          {
-            ...heatmap,
-            heatmap: {
-              ...heatmap.heatmap,
-              meta: {
-                type: undefined,
-              },
-            },
-          },
-          rawData
-        )
-      ).toThrow(HEATMAP_NOT_SCANLINES_ERROR);
-    });
-  });
-});

public/app/plugins/panel/heatmap-new/fields.ts

@@ -1,41 +1,33 @@
 import {
   DataFrame,
   DataFrameType,
-  Field,
   FieldType,
   formattedValueToString,
   getDisplayProcessor,
   getFieldDisplayName,
   getValueFormat,
   GrafanaTheme2,
-  incrRoundDn,
+  outerJoinDataFrames,
-  incrRoundUp,
   PanelData,
 } from '@grafana/data';
 import { calculateHeatmapFromData, bucketsToScanlines } from 'app/features/transformers/calculateHeatmap/heatmap';
 
-import { HeatmapSourceMode, PanelOptions } from './models.gen';
+import { HeatmapMode, PanelOptions } from './models.gen';
 
 export const enum BucketLayout {
   le = 'le',
   ge = 'ge',
+  unknown = 'unknown', // unknown
 }
 
-export interface HeatmapDataMapping {
-  lookup: Array<number[] | null>;
-  high: number[]; // index of values bigger than the max Y
-  low: number[]; // index of values less than the min Y
-}
-
-export const HEATMAP_NOT_SCANLINES_ERROR = 'A calculated heatmap was expected, but not found';
-
 export interface HeatmapData {
-  // List of heatmap frames
+  heatmap?: DataFrame; // data we will render
-  heatmap?: DataFrame;
+  exemplars?: DataFrame; // optionally linked exemplars
-  exemplars?: DataFrame;
+  exemplarColor?: string;
-  exemplarsMappings?: HeatmapDataMapping;
 
   yAxisValues?: Array<number | string | null>;
+  yLabelValues?: string[]; // matched ordinally to yAxisValues
+  matchByLabel?: string; // e.g. le, pod, etc.
 
   xBucketSize?: number;
   yBucketSize?: number;
@@ -54,113 +46,65 @@ export interface HeatmapData {
 }
 
 export function prepareHeatmapData(data: PanelData, options: PanelOptions, theme: GrafanaTheme2): HeatmapData {
-  const frames = data.series;
+  let frames = data.series;
   if (!frames?.length) {
     return {};
   }
 
-  const { source } = options;
+  const { mode } = options;
 
   const exemplars = data.annotations?.find((f) => f.name === 'exemplar');
 
-  if (source === HeatmapSourceMode.Calculate) {
+  if (mode === HeatmapMode.Calculate) {
     // TODO, check for error etc
-    return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), exemplars, theme);
+    return getHeatmapData(calculateHeatmapFromData(frames, options.calculate ?? {}), exemplars, theme);
   }
 
-  let sparseCellsHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapSparse);
+  // Check for known heatmap types
-  if (sparseCellsHeatmap) {
+  let bucketHeatmap: DataFrame | undefined = undefined;
-    return getSparseHeatmapData(sparseCellsHeatmap, exemplars, theme);
+  for (const frame of frames) {
+    switch (frame.meta?.type) {
+      case DataFrameType.HeatmapSparse:
+        return getSparseHeatmapData(frame, exemplars, theme);
+
+      case DataFrameType.HeatmapScanlines:
+        return getHeatmapData(frame, exemplars, theme);
+
+      case DataFrameType.HeatmapBuckets:
+        bucketHeatmap = frame; // the default format
+    }
   }
 
-  // Find a well defined heatmap
+  // Everything past here assumes a field for each row in the heatmap (buckets)
-  let scanlinesHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapScanlines);
+  if (!bucketHeatmap) {
-  if (scanlinesHeatmap) {
+    if (frames.length > 1) {
-    return getHeatmapData(scanlinesHeatmap, exemplars, theme);
+      bucketHeatmap = [
+        outerJoinDataFrames({
+          frames,
+        })!,
+      ][0];
+    } else {
+      bucketHeatmap = frames[0];
+    }
   }
 
-  let bucketsHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapBuckets);
+  // Some datasources return values in ascending order and require math to know the deltas
-  if (bucketsHeatmap) {
+  if (mode === HeatmapMode.Accumulated) {
+    console.log('TODO, deaccumulate the values');
+  }
+
+  const yFields = bucketHeatmap.fields.filter((f) => f.type === FieldType.number);
+  const matchByLabel = Object.keys(yFields[0].labels ?? {})[0];
+
+  const scanlinesFrame = bucketsToScanlines(bucketHeatmap);
   return {
-      yAxisValues: frames[0].fields.flatMap((field) =>
+    matchByLabel,
-        field.type === FieldType.number ? getFieldDisplayName(field) : []
+    yLabelValues: matchByLabel ? yFields.map((f) => f.labels?.[matchByLabel] ?? '') : undefined,
-      ),
+    yAxisValues: yFields.map((f) => getFieldDisplayName(f, bucketHeatmap, frames)),
-      ...getHeatmapData(bucketsToScanlines(bucketsHeatmap), exemplars, theme),
+    ...getHeatmapData(scanlinesFrame, exemplars, theme),
   };
-  }
-
-  if (source === HeatmapSourceMode.Data) {
-    let first = frames[0];
-    if (first.meta?.type !== DataFrameType.HeatmapScanlines) {
-      first = bucketsToScanlines(frames[0]);
-    }
-    return getHeatmapData(first, exemplars, theme);
-  }
-
-  // TODO, check for error etc
-  return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), exemplars, theme);
 }
 
-const getHeatmapFields = (dataFrame: DataFrame): Array<Field | undefined> => {
-  const xField: Field | undefined = dataFrame.fields.find((f) => f.name === 'xMin');
-  const yField: Field | undefined = dataFrame.fields.find((f) => f.name === 'yMin');
-  const countField: Field | undefined = dataFrame.fields.find((f) => f.name === 'count');
-
-  return [xField, yField, countField];
-};
-
-export const getExemplarsMapping = (heatmapData: HeatmapData, rawData: DataFrame): HeatmapDataMapping => {
-  if (heatmapData.heatmap?.meta?.type !== DataFrameType.HeatmapScanlines) {
-    throw HEATMAP_NOT_SCANLINES_ERROR;
-  }
-
-  const [fxs, fys] = getHeatmapFields(heatmapData.heatmap!);
-
-  if (!fxs || !fys) {
-    throw HEATMAP_NOT_SCANLINES_ERROR;
-  }
-
-  const mapping: HeatmapDataMapping = {
-    lookup: new Array(heatmapData.xBucketCount! * heatmapData.yBucketCount!).fill(null),
-    high: [],
-    low: [],
-  };
-
-  const xos: number[] | undefined = rawData.fields.find((f: Field) => f.type === 'time')?.values.toArray();
-  const yos: number[] | undefined = rawData.fields.find((f: Field) => f.type === 'number')?.values.toArray();
-
-  if (!xos || !yos) {
-    return mapping;
-  }
-
-  const xsmin = fxs.values.get(0);
-  const ysmin = fys.values.get(0);
-  const xsmax = fxs.values.get(fxs.values.length - 1) + heatmapData.xBucketSize!;
-  const ysmax = fys.values.get(fys.values.length - 1) + heatmapData.yBucketSize!;
-  xos.forEach((xo: number, i: number) => {
-    const yo = yos[i];
-    const xBucketIdx = Math.floor(incrRoundDn(incrRoundUp((xo - xsmin) / heatmapData.xBucketSize!, 1e-7), 1e-7));
-    const yBucketIdx = Math.floor(incrRoundDn(incrRoundUp((yo - ysmin) / heatmapData.yBucketSize!, 1e-7), 1e-7));
-
-    if (xo < xsmin || yo < ysmin) {
-      mapping.low.push(i);
-      return;
-    }
-
-    if (xo >= xsmax || yo >= ysmax) {
-      mapping.high.push(i);
-      return;
-    }
-
-    const index = xBucketIdx * heatmapData.yBucketCount! + yBucketIdx;
-    if (mapping.lookup[index] === null) {
-      mapping.lookup[index] = [];
-    }
-    mapping.lookup[index]?.push(i);
-  });
-  return mapping;
-};
-
 const getSparseHeatmapData = (
   frame: DataFrame,
   exemplars: DataFrame | undefined,
@@ -217,6 +161,9 @@ const getHeatmapData = (frame: DataFrame, exemplars: DataFrame | undefined, them
 
   // The "count" field
   const disp = frame.fields[2].display ?? getValueFormat('short');
+  const xName = frame.fields[0].name;
+  const yName = frame.fields[1].name;
+
   const data: HeatmapData = {
     heatmap: frame,
     exemplars,
@@ -226,16 +173,11 @@ const getHeatmapData = (frame: DataFrame, exemplars: DataFrame | undefined, them
     yBucketCount: yBinQty,
 
     // TODO: improve heuristic
-    xLayout: frame.fields[0].name === 'xMax' ? BucketLayout.le : BucketLayout.ge,
+    xLayout: xName === 'xMax' ? BucketLayout.le : xName === 'xMin' ? BucketLayout.ge : BucketLayout.unknown,
-    yLayout: frame.fields[1].name === 'yMax' ? BucketLayout.le : BucketLayout.ge,
+    yLayout: yName === 'yMax' ? BucketLayout.le : yName === 'yMin' ? BucketLayout.ge : BucketLayout.unknown,
 
     display: (v) => formattedValueToString(disp(v)),
   };
 
-  if (exemplars) {
-    data.exemplarsMappings = getExemplarsMapping(data, exemplars);
-    console.log('EXEMPLARS', data.exemplarsMappings, data.exemplars);
-  }
-
   return data;
 };

public/app/plugins/panel/heatmap-new/migrations.test.ts

@@ -25,6 +25,16 @@ describe('Heatmap Migrations', () => {
           "overrides": Array [],
         },
         "options": Object {
+          "calculate": Object {
+            "xAxis": Object {
+              "mode": "count",
+              "value": "100",
+            },
+            "yAxis": Object {
+              "mode": "count",
+              "value": "20",
+            },
+          },
           "cellGap": 2,
           "cellSize": 10,
           "color": Object {
@@ -35,21 +45,14 @@ describe('Heatmap Migrations', () => {
             "scheme": "BuGn",
             "steps": 256,
           },
-          "heatmap": Object {
+          "exemplars": Object {
-            "xAxis": Object {
+            "color": "rgba(255,0,255,0.7)",
-              "mode": "count",
-              "value": "100",
-            },
-            "yAxis": Object {
-              "mode": "count",
-              "value": "20",
-            },
           },
           "legend": Object {
             "show": true,
           },
+          "mode": "calculate",
           "showValue": "never",
-          "source": "calculate",
           "tooltip": Object {
             "show": true,
             "yHistogram": true,

public/app/plugins/panel/heatmap-new/migrations.ts

@@ -5,7 +5,7 @@ import {
   HeatmapCalculationOptions,
 } from 'app/features/transformers/calculateHeatmap/models.gen';
 
-import { HeatmapSourceMode, PanelOptions, defaultPanelOptions, HeatmapColorMode } from './models.gen';
+import { HeatmapMode, PanelOptions, defaultPanelOptions, HeatmapColorMode } from './models.gen';
 import { colorSchemes } from './palettes';
 
 /**
@@ -29,28 +29,28 @@ export function angularToReactHeatmap(angular: any): { fieldConfig: FieldConfigS
     overrides: [],
   };
 
-  const source = angular.dataFormat === 'tsbuckets' ? HeatmapSourceMode.Data : HeatmapSourceMode.Calculate;
+  const mode = angular.dataFormat === 'tsbuckets' ? HeatmapMode.Aggregated : HeatmapMode.Calculate;
-  const heatmap: HeatmapCalculationOptions = {
+  const calculate: HeatmapCalculationOptions = {
-    ...defaultPanelOptions.heatmap,
+    ...defaultPanelOptions.calculate,
   };
 
-  if (source === HeatmapSourceMode.Calculate) {
+  if (mode === HeatmapMode.Calculate) {
     if (angular.xBucketSize) {
-      heatmap.xAxis = { mode: HeatmapCalculationMode.Size, value: `${angular.xBucketSize}` };
+      calculate.xAxis = { mode: HeatmapCalculationMode.Size, value: `${angular.xBucketSize}` };
     } else if (angular.xBucketNumber) {
-      heatmap.xAxis = { mode: HeatmapCalculationMode.Count, value: `${angular.xBucketNumber}` };
+      calculate.xAxis = { mode: HeatmapCalculationMode.Count, value: `${angular.xBucketNumber}` };
     }
 
     if (angular.yBucketSize) {
-      heatmap.yAxis = { mode: HeatmapCalculationMode.Size, value: `${angular.yBucketSize}` };
+      calculate.yAxis = { mode: HeatmapCalculationMode.Size, value: `${angular.yBucketSize}` };
     } else if (angular.xBucketNumber) {
-      heatmap.yAxis = { mode: HeatmapCalculationMode.Count, value: `${angular.yBucketNumber}` };
+      calculate.yAxis = { mode: HeatmapCalculationMode.Count, value: `${angular.yBucketNumber}` };
     }
   }
 
   const options: PanelOptions = {
-    source,
+    mode,
-    heatmap,
+    calculate,
     color: {
       ...defaultPanelOptions.color,
       steps: 256, // best match with existing colors
@@ -67,6 +67,9 @@ export function angularToReactHeatmap(angular: any): { fieldConfig: FieldConfigS
       show: Boolean(angular.tooltip?.show),
       yHistogram: Boolean(angular.tooltip?.showHistogram),
     },
+    exemplars: {
+      ...defaultPanelOptions.exemplars,
+    },
   };
 
   // Migrate color options

public/app/plugins/panel/heatmap-new/models.gen.ts

@@ -8,10 +8,10 @@ import { HeatmapCalculationOptions } from 'app/features/transformers/calculateHe
 
 export const modelVersion = Object.freeze([1, 0]);
 
-export enum HeatmapSourceMode {
+export enum HeatmapMode {
-  Auto = 'auto',
+  Aggregated = 'agg',
   Calculate = 'calculate',
-  Data = 'data', // Use the data as is
+  Accumulated = 'acc', // accumulated
 }
 
 export enum HeatmapColorMode {
@@ -33,7 +33,6 @@ export interface HeatmapColorOptions {
   steps: number; // 2-128
 
   // Clamp the colors to the value range
-  field?: string;
   min?: number;
   max?: number;
 }
@@ -46,11 +45,15 @@ export interface HeatmapLegend {
   show: boolean;
 }
 
+export interface ExemplarConfig {
+  color: string;
+}
+
 export interface PanelOptions {
-  source: HeatmapSourceMode;
+  mode: HeatmapMode;
 
   color: HeatmapColorOptions;
-  heatmap?: HeatmapCalculationOptions;
+  calculate?: HeatmapCalculationOptions;
   showValue: VisibilityMode;
 
   cellGap?: number; // was cardPadding
@@ -62,10 +65,11 @@ export interface PanelOptions {
   legend: HeatmapLegend;
 
   tooltip: HeatmapTooltip;
+  exemplars: ExemplarConfig;
 }
 
 export const defaultPanelOptions: PanelOptions = {
-  source: HeatmapSourceMode.Auto,
+  mode: HeatmapMode.Aggregated,
   color: {
     mode: HeatmapColorMode.Scheme,
     scheme: 'Oranges',
@@ -82,6 +86,9 @@ export const defaultPanelOptions: PanelOptions = {
   legend: {
     show: true,
   },
+  exemplars: {
+    color: 'rgba(255,0,255,0.7)',
+  },
   cellGap: 1,
 };
 

public/app/plugins/panel/heatmap-new/module.tsx

@@ -1,6 +1,6 @@
 import React from 'react';
 
-import { Field, FieldConfigProperty, FieldType, PanelPlugin } from '@grafana/data';
+import { FieldConfigProperty, PanelPlugin } from '@grafana/data';
 import { config } from '@grafana/runtime';
 import { GraphFieldConfig } from '@grafana/schema';
 import { ColorScale } from 'app/core/components/ColorScale/ColorScale';
@@ -8,13 +8,7 @@ import { addHeatmapCalculationOptions } from 'app/features/transformers/calculat
 
 import { HeatmapPanel } from './HeatmapPanel';
 import { heatmapChangedHandler, heatmapMigrationHandler } from './migrations';
-import {
+import { PanelOptions, defaultPanelOptions, HeatmapMode, HeatmapColorMode, HeatmapColorScale } from './models.gen';
-  PanelOptions,
-  defaultPanelOptions,
-  HeatmapSourceMode,
-  HeatmapColorMode,
-  HeatmapColorScale,
-} from './models.gen';
 import { colorSchemes, quantizeScheme } from './palettes';
 import { HeatmapSuggestionsSupplier } from './suggestions';
 
@@ -30,36 +24,25 @@ export const plugin = new PanelPlugin<PanelOptions, GraphFieldConfig>(HeatmapPan
     let category = ['Heatmap'];
 
     builder.addRadio({
-      path: 'source',
+      path: 'mode',
-      name: 'Source',
+      name: 'Data',
-      defaultValue: HeatmapSourceMode.Auto,
+      defaultValue: defaultPanelOptions.mode,
       category,
       settings: {
         options: [
-          { label: 'Auto', value: HeatmapSourceMode.Auto },
+          { label: 'Aggregated', value: HeatmapMode.Aggregated },
-          { label: 'Calculate', value: HeatmapSourceMode.Calculate },
+          { label: 'Calculate', value: HeatmapMode.Calculate },
-          { label: 'Raw data', description: 'The results are already heatmap buckets', value: HeatmapSourceMode.Data },
+          //  { label: 'Accumulated', value: HeatmapMode.Accumulated, description: 'The query response values are accumulated' },
         ],
       },
     });
 
-    if (opts.source === HeatmapSourceMode.Calculate) {
+    if (opts.mode === HeatmapMode.Calculate) {
-      addHeatmapCalculationOptions('heatmap.', builder, opts.heatmap, category);
+      addHeatmapCalculationOptions('calculate.', builder, opts.calculate, category);
     }
 
     category = ['Colors'];
 
-    builder.addFieldNamePicker({
-      path: `color.field`,
-      name: 'Color with field',
-      category,
-      settings: {
-        filter: (f: Field) => f.type === FieldType.number,
-        noFieldsMessage: 'No numeric fields found',
-        placeholderText: 'Auto',
-      },
-    });
-
     builder.addRadio({
       path: `color.mode`,
       name: 'Mode',
@@ -84,7 +67,6 @@ export const plugin = new PanelPlugin<PanelOptions, GraphFieldConfig>(HeatmapPan
     builder.addRadio({
       path: `color.scale`,
       name: 'Scale',
-      description: '',
       defaultValue: defaultPanelOptions.color.scale,
       category,
       settings: {
@@ -141,7 +123,7 @@ export const plugin = new PanelPlugin<PanelOptions, GraphFieldConfig>(HeatmapPan
       .addCustomEditor({
         id: '__scale__',
         path: `__scale__`,
-        name: 'Scale',
+        name: '',
         category,
         editor: () => {
           const palette = quantizeScheme(opts.color, config.theme2);
@@ -240,5 +222,13 @@ export const plugin = new PanelPlugin<PanelOptions, GraphFieldConfig>(HeatmapPan
       defaultValue: defaultPanelOptions.legend.show,
       category,
     });
+
+    category = ['Exemplars'];
+    builder.addColorPicker({
+      path: 'exemplars.color',
+      name: 'Color',
+      defaultValue: defaultPanelOptions.exemplars.color,
+      category,
+    });
   })
   .setSuggestionsSupplier(new HeatmapSuggestionsSupplier());

public/app/plugins/panel/heatmap-new/utils.ts

@@ -13,8 +13,8 @@ interface PathbuilderOpts {
   each: (u: uPlot, seriesIdx: number, dataIdx: number, lft: number, top: number, wid: number, hgt: number) => void;
   gap?: number | null;
   hideThreshold?: number;
-  xCeil?: boolean;
+  xAlign?: -1 | 0 | 1;
-  yCeil?: boolean;
+  yAlign?: -1 | 0 | 1;
   disp: {
     fill: {
       values: (u: uPlot, seriesIndex: number) => number[];
@@ -23,8 +23,13 @@ interface PathbuilderOpts {
   };
 }
 
+interface PointsBuilderOpts {
+  each: (u: uPlot, seriesIdx: number, dataIdx: number, lft: number, top: number, wid: number, hgt: number) => void;
+}
+
 export interface HeatmapHoverEvent {
-  index: number;
+  seriesIdx: number;
+  dataIdx: number;
   pageX: number;
   pageY: number;
 }
@@ -44,6 +49,7 @@ interface PrepConfigOpts {
   timeZone: string;
   getTimeRange: () => TimeRange;
   palette: string[];
+  exemplarColor: string;
   cellGap?: number | null; // in css pixels
   hideThreshold?: number;
 }
@@ -66,6 +72,7 @@ export function prepConfig(opts: PrepConfigOpts) {
   const pxRatio = devicePixelRatio;
 
   let heatmapType = dataRef.current?.heatmap?.meta?.type;
+  const exemplarFillColor = theme.visualization.getColorByName(opts.exemplarColor);
 
   let qt: Quadtree;
   let hRect: Rect | null;
@@ -143,7 +150,8 @@ export function prepConfig(opts: PrepConfigOpts) {
             }
 
             onhover({
-              index: sel,
+              seriesIdx: i,
+              dataIdx: sel,
               pageX: rect.left + u.cursor.left!,
               pageY: rect.top + u.cursor.top!,
             });
@@ -209,13 +217,16 @@ export function prepConfig(opts: PrepConfigOpts) {
     range: shouldUseLogScale
       ? undefined
       : (u, dataMin, dataMax) => {
-          let bucketSize = dataRef.current?.yBucketSize;
+          const bucketSize = dataRef.current?.yBucketSize;
 
           if (bucketSize) {
             if (dataRef.current?.yLayout === BucketLayout.le) {
               dataMin -= bucketSize!;
-            } else {
+            } else if (dataRef.current?.yLayout === BucketLayout.ge) {
               dataMax += bucketSize!;
+            } else {
+              dataMin -= bucketSize! / 2;
+              dataMax += bucketSize! / 2;
             }
           } else {
             // how to expand scale range if inferred non-regular or log buckets?
@@ -233,10 +244,10 @@ export function prepConfig(opts: PrepConfigOpts) {
     theme: theme,
     splits: hasLabeledY
       ? () => {
-          let ys = dataRef.current?.heatmap?.fields[1].values.toArray()!;
+          const ys = dataRef.current?.heatmap?.fields[1].values.toArray()!;
-          let splits = ys.slice(0, ys.length - ys.lastIndexOf(ys[0]));
+          const splits = ys.slice(0, ys.length - ys.lastIndexOf(ys[0]));
 
-          let bucketSize = dataRef.current?.yBucketSize!;
+          const bucketSize = dataRef.current?.yBucketSize!;
 
           if (dataRef.current?.yLayout === BucketLayout.le) {
             splits.unshift(ys[0] - bucketSize);
@@ -249,11 +260,11 @@ export function prepConfig(opts: PrepConfigOpts) {
       : undefined,
     values: hasLabeledY
       ? () => {
-          let yAxisValues = dataRef.current?.yAxisValues?.slice()!;
+          const yAxisValues = dataRef.current?.yAxisValues?.slice()!;
 
           if (dataRef.current?.yLayout === BucketLayout.le) {
             yAxisValues.unshift('0.0'); // assumes dense layout where lowest bucket's low bound is 0-ish
-          } else {
+          } else if (dataRef.current?.yLayout === BucketLayout.ge) {
             yAxisValues.push('+Inf');
           }
 
@@ -262,8 +273,9 @@ export function prepConfig(opts: PrepConfigOpts) {
       : undefined,
   });
 
-  let pathBuilder = heatmapType === DataFrameType.HeatmapScanlines ? heatmapPathsDense : heatmapPathsSparse;
+  const pathBuilder = heatmapType === DataFrameType.HeatmapScanlines ? heatmapPathsDense : heatmapPathsSparse;
 
+  // heatmap layer
   builder.addSeries({
     facets: [
       {
@@ -289,8 +301,8 @@ export function prepConfig(opts: PrepConfigOpts) {
       },
       gap: cellGap,
       hideThreshold,
-      xCeil: dataRef.current?.xLayout === BucketLayout.le,
+      xAlign: dataRef.current?.xLayout === BucketLayout.le ? -1 : dataRef.current?.xLayout === BucketLayout.ge ? 1 : 0,
-      yCeil: dataRef.current?.yLayout === BucketLayout.le,
+      yAlign: dataRef.current?.yLayout === BucketLayout.le ? -1 : dataRef.current?.yLayout === BucketLayout.ge ? 1 : 0,
       disp: {
         fill: {
           values: (u, seriesIdx) => {
@@ -305,6 +317,38 @@ export function prepConfig(opts: PrepConfigOpts) {
     scaleKey: '', // facets' scales used (above)
   });
 
+  // exemplars layer
+  builder.addSeries({
+    facets: [
+      {
+        scale: 'x',
+        auto: true,
+        sorted: 1,
+      },
+      {
+        scale: 'y',
+        auto: true,
+      },
+    ],
+    pathBuilder: heatmapPathsPoints(
+      {
+        each: (u, seriesIdx, dataIdx, x, y, xSize, ySize) => {
+          qt.add({
+            x: x - u.bbox.left,
+            y: y - u.bbox.top,
+            w: xSize,
+            h: ySize,
+            sidx: seriesIdx,
+            didx: dataIdx,
+          });
+        },
+      },
+      exemplarFillColor
+    ) as any,
+    theme,
+    scaleKey: '', // facets' scales used (above)
+  });
+
   builder.setCursor({
     drag: {
       x: true,
@@ -348,7 +392,7 @@ export function prepConfig(opts: PrepConfigOpts) {
 const CRISP_EDGES_GAP_MIN = 4;
 
 export function heatmapPathsDense(opts: PathbuilderOpts) {
-  const { disp, each, gap = 1, hideThreshold = 0, xCeil = false, yCeil = false } = opts;
+  const { disp, each, gap = 1, hideThreshold = 0, xAlign = 1, yAlign = 1 } = opts;
 
   const pxRatio = devicePixelRatio;
 
@@ -408,8 +452,8 @@ export function heatmapPathsDense(opts: PathbuilderOpts) {
         // let xCeil = false;
         // let yCeil = false;
 
-        let xOffset = xCeil ? -xSize : 0;
+        let xOffset = xAlign === -1 ? -xSize : xAlign === 0 ? -xSize / 2 : 0;
-        let yOffset = yCeil ? 0 : -ySize;
+        let yOffset = yAlign === 1 ? -ySize : yAlign === 0 ? -ySize / 2 : 0;
 
         // pre-compute x and y offsets
         let cys = ys.slice(0, yBinQty).map((y) => round(valToPosY(y, scaleY, yDim, yOff) + yOffset));
@@ -453,6 +497,65 @@ export function heatmapPathsDense(opts: PathbuilderOpts) {
   };
 }
 
+export function heatmapPathsPoints(opts: PointsBuilderOpts, exemplarColor: string) {
+  return (u: uPlot, seriesIdx: number) => {
+    uPlot.orient(
+      u,
+      seriesIdx,
+      (
+        series,
+        dataX,
+        dataY,
+        scaleX,
+        scaleY,
+        valToPosX,
+        valToPosY,
+        xOff,
+        yOff,
+        xDim,
+        yDim,
+        moveTo,
+        lineTo,
+        rect,
+        arc
+      ) => {
+        //console.time('heatmapPathsSparse');
+
+        [dataX, dataY] = dataY as unknown as number[][];
+
+        let points = new Path2D();
+        let fillPaths = [points];
+        let fillPalette = [exemplarColor ?? 'rgba(255,0,255,0.7)'];
+
+        for (let i = 0; i < dataX.length; i++) {
+          let yVal = dataY[i]!;
+          yVal -= 0.5; // center vertically in bucket (when tiles are le)
+          // y-randomize vertically to distribute exemplars in same bucket at same time
+          let randSign = Math.round(Math.random()) * 2 - 1;
+          yVal += randSign * 0.5 * Math.random();
+
+          let x = valToPosX(dataX[i], scaleX, xDim, xOff);
+          let y = valToPosY(yVal, scaleY, yDim, yOff);
+          let w = 8;
+          let h = 8;
+
+          rect(points, x - w / 2, y - h / 2, w, h);
+
+          opts.each(u, seriesIdx, i, x - w / 2, y - h / 2, w, h);
+        }
+
+        u.ctx.save();
+        u.ctx.rect(u.bbox.left, u.bbox.top, u.bbox.width, u.bbox.height);
+        u.ctx.clip();
+        fillPaths.forEach((p, i) => {
+          u.ctx.fillStyle = fillPalette[i];
+          u.ctx.fill(p);
+        });
+        u.ctx.restore();
+      }
+    );
+  };
+}
 // accepts xMax, yMin, yMax, count
 // xbinsize? x tile sizes are uniform?
 export function heatmapPathsSparse(opts: PathbuilderOpts) {