HeatmapNG: Sparse renderer (#48993)

Co-authored-by: Ryan McKinley <ryantxu@gmail.com>

public/app/plugins/panel/heatmap-new/HeatmapHoverView.tsx

@@ -1,9 +1,19 @@
 import React, { useEffect, useRef } from 'react';
 
-import { DataFrameView, Field, FieldType, formattedValueToString, getFieldDisplayName, LinkModel } from '@grafana/data';
+import {
+  DataFrameType,
+  DataFrameView,
+  Field,
+  FieldType,
+  formattedValueToString,
+  getFieldDisplayName,
+  LinkModel,
+} from '@grafana/data';
 import { LinkButton, VerticalGroup } from '@grafana/ui';
 import { getDashboardSrv } from 'app/features/dashboard/services/DashboardSrv';
 
+import { DataHoverView } from '../geomap/components/DataHoverView';
+
 import { BucketLayout, HeatmapData } from './fields';
 import { HeatmapHoverEvent } from './utils';
 
@@ -177,6 +187,14 @@ export const HeatmapHoverView = ({ data, hover, showHistogram }: Props) => {
     return <div>EXEMPLARS: {JSON.stringify(exemplarIndex)}</div>;
   };
 
+  if (data.heatmap?.meta?.type === DataFrameType.HeatmapSparse) {
+    return (
+      <div>
+        <DataHoverView data={data.heatmap} rowIndex={hover.index} />
+      </div>
+    );
+  }
+
   return (
     <>
       <div>

public/app/plugins/panel/heatmap-new/HeatmapPanel.tsx

@@ -1,7 +1,7 @@
 import { css } from '@emotion/css';
 import React, { useCallback, useMemo, useRef, useState } from 'react';
 
-import { GrafanaTheme2, PanelProps, reduceField, ReducerID, TimeRange } from '@grafana/data';
+import { DataFrameType, GrafanaTheme2, PanelProps, reduceField, ReducerID, TimeRange } from '@grafana/data';
 import { PanelDataErrorView } from '@grafana/runtime';
 import {
   Portal,
@@ -16,7 +16,7 @@ import { CloseButton } from 'app/core/components/CloseButton/CloseButton';
 import { ColorScale } from 'app/core/components/ColorScale/ColorScale';
 
 import { HeatmapHoverView } from './HeatmapHoverView';
-import { HeatmapData, prepareHeatmapData } from './fields';
+import { prepareHeatmapData } from './fields';
 import { PanelOptions } from './models.gen';
 import { quantizeScheme } from './palettes';
 import { HeatmapHoverEvent, prepConfig } from './utils';
@@ -76,7 +76,7 @@ export const HeatmapPanel: React.FC<HeatmapPanelProps> = ({
   );
 
   // ugh
-  const dataRef = useRef<HeatmapData>(info);
+  const dataRef = useRef(info);
   dataRef.current = info;
 
   const builder = useMemo(() => {
@@ -103,13 +103,15 @@ export const HeatmapPanel: React.FC<HeatmapPanelProps> = ({
       return null;
     }
 
-    const field = info.heatmap.fields[2];
-    const { min, max } = reduceField({ field, reducers: [ReducerID.min, ReducerID.max] });
+    let heatmapType = dataRef.current?.heatmap?.meta?.type;
+    let countFieldIdx = heatmapType === DataFrameType.HeatmapScanlines ? 2 : 3;
+    const countField = info.heatmap.fields[countFieldIdx];
+
+    const { min, max } = reduceField({ field: countField, reducers: [ReducerID.min, ReducerID.max] });
 
     let hoverValue: number | undefined = undefined;
     if (hover && info.heatmap.fields) {
-      const countField = info.heatmap.fields[2];
-      hoverValue = countField?.values.get(hover.index);
+      hoverValue = countField.values.get(hover.index);
     }
 
     return (

public/app/plugins/panel/heatmap-new/fields.ts

@@ -68,6 +68,11 @@ export function prepareHeatmapData(data: PanelData, options: PanelOptions, theme
     return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), exemplars, theme);
   }
 
+  let sparseCellsHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapSparse);
+  if (sparseCellsHeatmap) {
+    return getSparseHeatmapData(sparseCellsHeatmap, exemplars, theme);
+  }
+
   // Find a well defined heatmap
   let scanlinesHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapScanlines);
   if (scanlinesHeatmap) {
@@ -84,12 +89,8 @@ export function prepareHeatmapData(data: PanelData, options: PanelOptions, theme
     };
   }
 
-  let first = frames[0];
-  if (first.meta?.type === DataFrameType.HeatmapSparse) {
-    return getSparseHeatmapData(first, exemplars, theme);
-  }
-
   if (source === HeatmapSourceMode.Data) {
+    let first = frames[0];
     if (first.meta?.type !== DataFrameType.HeatmapScanlines) {
       first = bucketsToScanlines(frames[0]);
     }

public/app/plugins/panel/heatmap-new/models.gen.ts

@@ -30,7 +30,7 @@ export interface HeatmapColorOptions {
   fill: string; // when opacity mode, the target color
   scale: HeatmapColorScale; // for opacity mode
   exponent: number; // when scale== sqrt
-  steps: number; // 2-256
+  steps: number; // 2-128
 
   // Clamp the colors to the value range
   field?: string;
@@ -80,7 +80,7 @@ export const defaultPanelOptions: PanelOptions = {
     show: true,
     yHistogram: false,
   },
-  cellGap: 3,
+  cellGap: 1,
 };
 
 export interface PanelFieldConfig extends HideableFieldConfig {

public/app/plugins/panel/heatmap-new/utils.ts

@@ -1,8 +1,8 @@
 import { MutableRefObject, RefObject } from 'react';
 import uPlot from 'uplot';
 
-import { GrafanaTheme2, TimeRange } from '@grafana/data';
-import { AxisPlacement, ScaleDirection, ScaleOrientation } from '@grafana/schema';
+import { DataFrameType, GrafanaTheme2, TimeRange } from '@grafana/data';
+import { AxisPlacement, ScaleDirection, ScaleDistribution, ScaleOrientation } from '@grafana/schema';
 import { UPlotConfigBuilder } from '@grafana/ui';
 
 import { pointWithin, Quadtree, Rect } from '../barchart/quadtree';
@@ -63,6 +63,10 @@ export function prepConfig(opts: PrepConfigOpts) {
     hideThreshold,
   } = opts;
 
+  const pxRatio = devicePixelRatio;
+
+  let heatmapType = dataRef.current?.heatmap?.meta?.type;
+
   let qt: Quadtree;
   let hRect: Rect | null;
 
@@ -191,20 +195,31 @@ export function prepConfig(opts: PrepConfigOpts) {
     theme: theme,
   });
 
+  const shouldUseLogScale = heatmapType === DataFrameType.HeatmapSparse;
+
   builder.addScale({
     scaleKey: 'y',
     isTime: false,
     // distribution: ScaleDistribution.Ordinal, // does not work with facets/scatter yet
     orientation: ScaleOrientation.Vertical,
     direction: ScaleDirection.Up,
-    range: (u, dataMin, dataMax) => {
+    // should be tweakable manually
+    distribution: shouldUseLogScale ? ScaleDistribution.Log : ScaleDistribution.Linear,
+    log: 2,
+    range: shouldUseLogScale
+      ? undefined
+      : (u, dataMin, dataMax) => {
           let bucketSize = dataRef.current?.yBucketSize;
 
+          if (bucketSize) {
             if (dataRef.current?.yLayout === BucketLayout.le) {
               dataMin -= bucketSize!;
             } else {
               dataMax += bucketSize!;
             }
+          } else {
+            // how to expand scale range if inferred non-regular or log buckets?
+          }
 
           return [dataMin, dataMax];
         },
@@ -247,6 +262,8 @@ export function prepConfig(opts: PrepConfigOpts) {
       : undefined,
   });
 
+  let pathBuilder = heatmapType === DataFrameType.HeatmapScanlines ? heatmapPathsDense : heatmapPathsSparse;
+
   builder.addSeries({
     facets: [
       {
@@ -259,7 +276,7 @@ export function prepConfig(opts: PrepConfigOpts) {
         auto: true,
       },
     ],
-    pathBuilder: heatmapPaths({
+    pathBuilder: pathBuilder({
       each: (u, seriesIdx, dataIdx, x, y, xSize, ySize) => {
         qt.add({
           x: x - u.bbox.left,
@@ -276,7 +293,10 @@ export function prepConfig(opts: PrepConfigOpts) {
       yCeil: dataRef.current?.yLayout === BucketLayout.le,
       disp: {
         fill: {
-          values: (u, seriesIdx) => countsToFills(u, seriesIdx, palette),
+          values: (u, seriesIdx) => {
+            let countFacetIdx = heatmapType === DataFrameType.HeatmapScanlines ? 2 : 3;
+            return countsToFills(u.data[seriesIdx][countFacetIdx] as unknown as number[], palette);
+          },
           index: palette,
         },
       },
@@ -295,8 +315,8 @@ export function prepConfig(opts: PrepConfigOpts) {
       if (seriesIdx === 1) {
         hRect = null;
 
-        let cx = u.cursor.left! * devicePixelRatio;
-        let cy = u.cursor.top! * devicePixelRatio;
+        let cx = u.cursor.left! * pxRatio;
+        let cy = u.cursor.top! * pxRatio;
 
         qt.get(cx, cy, 1, 1, (o) => {
           if (pointWithin(cx, cy, o.x, o.y, o.x + o.w, o.y + o.h)) {
@@ -313,10 +333,10 @@ export function prepConfig(opts: PrepConfigOpts) {
         let isHovered = hRect && seriesIdx === hRect.sidx;
 
         return {
-          left: isHovered ? hRect!.x / devicePixelRatio : -10,
-          top: isHovered ? hRect!.y / devicePixelRatio : -10,
-          width: isHovered ? hRect!.w / devicePixelRatio : 0,
-          height: isHovered ? hRect!.h / devicePixelRatio : 0,
+          left: isHovered ? hRect!.x / pxRatio : -10,
+          top: isHovered ? hRect!.y / pxRatio : -10,
+          width: isHovered ? hRect!.w / pxRatio : 0,
+          height: isHovered ? hRect!.h / pxRatio : 0,
         };
       },
     },
@@ -325,8 +345,16 @@ export function prepConfig(opts: PrepConfigOpts) {
   return builder;
 }
 
-export function heatmapPaths(opts: PathbuilderOpts) {
-  const { disp, each, gap, hideThreshold = 0, xCeil = false, yCeil = false } = opts;
+const CRISP_EDGES_GAP_MIN = 4;
+
+export function heatmapPathsDense(opts: PathbuilderOpts) {
+  const { disp, each, gap = 1, hideThreshold = 0, xCeil = false, yCeil = false } = opts;
+
+  const pxRatio = devicePixelRatio;
+
+  const round = gap! >= CRISP_EDGES_GAP_MIN ? Math.round : (v: number) => v;
+
+  const cellGap = Math.round(gap! * pxRatio);
 
   return (u: uPlot, seriesIdx: number) => {
     uPlot.orient(
@@ -372,15 +400,9 @@ export function heatmapPaths(opts: PathbuilderOpts) {
         let xSize = Math.abs(valToPosX(xBinIncr, scaleX, xDim, xOff) - valToPosX(0, scaleX, xDim, xOff));
         let ySize = Math.abs(valToPosY(yBinIncr, scaleY, yDim, yOff) - valToPosY(0, scaleY, yDim, yOff));
 
-        const autoGapFactor = 0.05;
-
-        // tile gap control
-        let xGap = gap != null ? gap * devicePixelRatio : Math.max(0, autoGapFactor * Math.min(xSize, ySize));
-        let yGap = xGap;
-
         // clamp min tile size to 1px
-        xSize = Math.max(1, Math.round(xSize - xGap));
-        ySize = Math.max(1, Math.round(ySize - yGap));
+        xSize = Math.max(1, round(xSize - cellGap));
+        ySize = Math.max(1, round(ySize - cellGap));
 
         // bucket agg direction
         // let xCeil = false;
@@ -390,9 +412,9 @@ export function heatmapPaths(opts: PathbuilderOpts) {
         let yOffset = yCeil ? 0 : -ySize;
 
         // pre-compute x and y offsets
-        let cys = ys.slice(0, yBinQty).map((y) => Math.round(valToPosY(y, scaleY, yDim, yOff) + yOffset));
+        let cys = ys.slice(0, yBinQty).map((y) => round(valToPosY(y, scaleY, yDim, yOff) + yOffset));
         let cxs = Array.from({ length: xBinQty }, (v, i) =>
-          Math.round(valToPosX(xs[i * yBinQty], scaleX, xDim, xOff) + xOffset)
+          round(valToPosX(xs[i * yBinQty], scaleX, xDim, xOff) + xOffset)
         );
 
         for (let i = 0; i < dlen; i++) {
@@ -431,9 +453,136 @@ export function heatmapPaths(opts: PathbuilderOpts) {
   };
 }
 
-export const countsToFills = (u: uPlot, seriesIdx: number, palette: string[]) => {
-  let counts = u.data[seriesIdx][2] as unknown as number[];
+// accepts xMax, yMin, yMax, count
+// xbinsize? x tile sizes are uniform?
+export function heatmapPathsSparse(opts: PathbuilderOpts) {
+  const { disp, each, gap = 1, hideThreshold = 0 } = opts;
 
+  const pxRatio = devicePixelRatio;
+
+  const round = gap! >= CRISP_EDGES_GAP_MIN ? Math.round : (v: number) => v;
+
+  const cellGap = Math.round(gap! * pxRatio);
+
+  return (u: uPlot, seriesIdx: number) => {
+    uPlot.orient(
+      u,
+      seriesIdx,
+      (
+        series,
+        dataX,
+        dataY,
+        scaleX,
+        scaleY,
+        valToPosX,
+        valToPosY,
+        xOff,
+        yOff,
+        xDim,
+        yDim,
+        moveTo,
+        lineTo,
+        rect,
+        arc
+      ) => {
+        //console.time('heatmapPathsSparse');
+
+        let d = u.data[seriesIdx];
+        const xMaxs = d[0] as unknown as number[]; // xMax, do we get interval?
+        const yMins = d[1] as unknown as number[];
+        const yMaxs = d[2] as unknown as number[];
+        const counts = d[3] as unknown as number[];
+        const dlen = xMaxs.length;
+
+        // fill colors are mapped from interpolating densities / counts along some gradient
+        // (should be quantized to 64 colors/levels max. e.g. 16)
+        let fills = disp.fill.values(u, seriesIdx);
+        let fillPalette = disp.fill.index ?? [...new Set(fills)];
+
+        let fillPaths = fillPalette.map((color) => new Path2D());
+
+        // cache all tile bounds
+        let xOffs = new Map();
+        let yOffs = new Map();
+
+        for (let i = 0; i < xMaxs.length; i++) {
+          let xMax = xMaxs[i];
+          let yMin = yMins[i];
+          let yMax = yMaxs[i];
+
+          if (!xOffs.has(xMax)) {
+            xOffs.set(xMax, round(valToPosX(xMax, scaleX, xDim, xOff)));
+          }
+
+          if (!yOffs.has(yMin)) {
+            yOffs.set(yMin, round(valToPosY(yMin, scaleY, yDim, yOff)));
+          }
+
+          if (!yOffs.has(yMax)) {
+            yOffs.set(yMax, round(valToPosY(yMax, scaleY, yDim, yOff)));
+          }
+        }
+
+        // uniform x size (interval, step)
+        let xSizeUniform = xOffs.get(xMaxs.find((v) => v !== xMaxs[0])) - xOffs.get(xMaxs[0]);
+
+        for (let i = 0; i < dlen; i++) {
+          if (counts[i] <= hideThreshold) {
+            continue;
+          }
+
+          let xMax = xMaxs[i];
+          let yMin = yMins[i];
+          let yMax = yMaxs[i];
+
+          let xMaxPx = xOffs.get(xMax); // xSize is from interval, or inferred delta?
+          let yMinPx = yOffs.get(yMin);
+          let yMaxPx = yOffs.get(yMax);
+
+          let xSize = xSizeUniform;
+          let ySize = yMinPx - yMaxPx;
+
+          // clamp min tile size to 1px
+          xSize = Math.max(1, xSize - cellGap);
+          ySize = Math.max(1, ySize - cellGap);
+
+          let x = xMaxPx;
+          let y = yMinPx;
+
+          // filter out 0 counts and out of view
+          // if (
+          //   xs[i] + xBinIncr >= scaleX.min! &&
+          //   xs[i] - xBinIncr <= scaleX.max! &&
+          //   ys[i] + yBinIncr >= scaleY.min! &&
+          //   ys[i] - yBinIncr <= scaleY.max!
+          // ) {
+          let fillPath = fillPaths[fills[i]];
+
+          rect(fillPath, x, y, xSize, ySize);
+
+          each(u, 1, i, x, y, xSize, ySize);
+          //  }
+        }
+
+        u.ctx.save();
+        //	u.ctx.globalAlpha = 0.8;
+        u.ctx.rect(u.bbox.left, u.bbox.top, u.bbox.width, u.bbox.height);
+        u.ctx.clip();
+        fillPaths.forEach((p, i) => {
+          u.ctx.fillStyle = fillPalette[i];
+          u.ctx.fill(p);
+        });
+        u.ctx.restore();
+
+        //console.timeEnd('heatmapPathsSparse');
+
+        return null;
+      }
+    );
+  };
+}
+
+export const countsToFills = (counts: number[], palette: string[]) => {
   // TODO: integrate 1e-9 hideThreshold?
   const hideThreshold = 0;