From 454e8046573752d67371ca7493c211f762133de3 Mon Sep 17 00:00:00 2001 From: Stephanie Closson Date: Thu, 5 May 2022 19:27:28 -0300 Subject: [PATCH] Heatmap (new): add exemplar mapping function (#48780) --- .../panel/heatmap-new/HeatmapPanel.tsx | 2 +- .../plugins/panel/heatmap-new/fields.test.ts | 312 +++++++++++++++++- .../app/plugins/panel/heatmap-new/fields.ts | 105 +++++- .../plugins/panel/heatmap-new/suggestions.ts | 2 +- .../plugins/panel/heatmap-new/utils.test.ts | 5 + 5 files changed, 411 insertions(+), 15 deletions(-) create mode 100644 public/app/plugins/panel/heatmap-new/utils.test.ts diff --git a/public/app/plugins/panel/heatmap-new/HeatmapPanel.tsx b/public/app/plugins/panel/heatmap-new/HeatmapPanel.tsx index d1cf7093238..7814e421a1e 100644 --- a/public/app/plugins/panel/heatmap-new/HeatmapPanel.tsx +++ b/public/app/plugins/panel/heatmap-new/HeatmapPanel.tsx @@ -42,7 +42,7 @@ export const HeatmapPanel: React.FC = ({ let timeRangeRef = useRef(timeRange); timeRangeRef.current = timeRange; - const info = useMemo(() => prepareHeatmapData(data.series, options, theme), [data, options, theme]); + const info = useMemo(() => prepareHeatmapData(data, options, theme), [data, options, theme]); const facets = useMemo(() => [null, info.heatmap?.fields.map((f) => f.values.toArray())], [info.heatmap]); diff --git a/public/app/plugins/panel/heatmap-new/fields.test.ts b/public/app/plugins/panel/heatmap-new/fields.test.ts index 7bc2e1b1441..ce8be363bfa 100644 --- a/public/app/plugins/panel/heatmap-new/fields.test.ts +++ b/public/app/plugins/panel/heatmap-new/fields.test.ts @@ -1,5 +1,6 @@ -import { createTheme } from '@grafana/data'; +import { createTheme, ArrayVector, DataFrameType, FieldType } from '@grafana/data'; +import { BucketLayout, getAnnotationMapping, HEATMAP_NOT_SCANLINES_ERROR } from './fields'; import { PanelOptions } from './models.gen'; const theme = createTheme(); @@ -12,3 +13,312 @@ describe('Heatmap data', () => { expect(options).toBeDefined(); }); }); + +describe('creating a heatmap data mapping', () => { + describe('generates a simple data mapping with orderly data', () => { + const mapping = getAnnotationMapping( + { + heatmap: { + name: 'test', + meta: { + type: DataFrameType.HeatmapScanlines, + }, + fields: [ + { + name: 'xMin', + type: FieldType.number, + config: {}, + values: new ArrayVector([1]), + }, + { + name: 'yMin', + type: FieldType.number, + config: {}, + values: new ArrayVector([1, 4, 7]), + }, + { + name: 'count', + type: FieldType.number, + config: {}, + values: new ArrayVector([3, 3, 3]), + }, + ], + length: 3, + }, + xBucketCount: 1, + xBucketSize: 9, + xLayout: BucketLayout.ge, + yBucketCount: 3, + yBucketSize: 3, + yLayout: BucketLayout.ge, + }, + { + name: 'origdata', + fields: [ + { + name: 'time', + type: FieldType.time, + config: {}, + values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]), + }, + { + name: 'value', + type: FieldType.number, + config: {}, + values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]), + }, + ], + length: 2, + } + ); + + it('takes good data, and delivers a working data mapping', () => { + expect(mapping.lookup.length).toEqual(3); + expect(mapping.lookup[0]).toEqual([0, 1, 2]); + expect(mapping.lookup[1]).toEqual([3, 4, 5]); + expect(mapping.lookup[2]).toEqual([6, 7, 8]); + }); + }); + + describe('generates a data mapping with less orderly data (counts are different)', () => { + const heatmap = { + heatmap: { + name: 'test', + meta: { + type: DataFrameType.HeatmapScanlines, + }, + fields: [ + { + name: 'xMin', + type: FieldType.number, + config: {}, + values: new ArrayVector([1]), + }, + { + name: 'yMin', + type: FieldType.number, + config: {}, + values: new ArrayVector([1, 4, 7]), + }, + { + name: 'count', + type: FieldType.number, + config: {}, + values: new ArrayVector([2, 1, 6]), + }, + ], + length: 3, + }, + xBucketCount: 1, + xBucketSize: 9, + xLayout: BucketLayout.ge, + yBucketCount: 3, + yBucketSize: 3, + yLayout: BucketLayout.ge, + }; + const rawData = { + name: 'origdata', + fields: [ + { + name: 'time', + type: FieldType.time, + config: {}, + values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]), + }, + { + name: 'value', + type: FieldType.number, + config: {}, + values: new ArrayVector([7, 1, 8, 7, 2, 7, 8, 8, 4]), + }, + ], + length: 2, + }; + + it("Puts data into the proper buckets, when we don't care about the count", () => { + // In this case, we are just finding proper values, but don't care if a values + // exists in the bucket in the original data or not. Therefore, we should see + // a value mapped into the second mapping bucket containing the value '8'. + const mapping = getAnnotationMapping(heatmap, rawData); + expect(mapping.lookup.length).toEqual(3); + expect(mapping.lookup[0]).toEqual([1, 4]); + expect(mapping.lookup[1]).toEqual([8]); + expect(mapping.lookup[2]).toEqual([0, 2, 3, 5, 6, 7]); + }); + }); + + describe('Handles a larger data set that will not fill all buckets', () => { + const mapping = getAnnotationMapping( + { + heatmap: { + name: 'test', + meta: { + type: DataFrameType.HeatmapScanlines, + }, + fields: [ + { + name: 'xMin', + type: FieldType.number, + config: {}, + values: new ArrayVector([1, 1, 1, 4, 4, 4, 7, 7, 7]), + }, + { + name: 'yMin', + type: FieldType.number, + config: {}, + values: new ArrayVector([1, 4, 7, 1, 4, 7, 1, 4, 7]), + }, + { + name: 'count', + type: FieldType.number, + config: {}, + values: new ArrayVector([0, 0, 2, 0, 0, 3, 0, 2, 0]), + }, + ], + length: 3, + }, + xBucketCount: 3, + xBucketSize: 3, + xLayout: BucketLayout.ge, + yBucketCount: 3, + yBucketSize: 3, + yLayout: BucketLayout.ge, + }, + { + name: 'origdata', + fields: [ + { + name: 'time', + type: FieldType.time, + config: {}, + values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]), + }, + { + name: 'value', + type: FieldType.number, + config: {}, + values: new ArrayVector([8, 0, 8, 7, 7, 8, 6, 10, 6]), + }, + ], + length: 2, + } + ); + + it('Creates the data mapping correctly', () => { + expect(mapping.lookup.length).toEqual(9); + expect(mapping).toEqual({ + lookup: [null, null, [0, 2], null, null, [3, 4, 5], null, [6, 8], null], + low: [1], + high: [7], + }); + }); + + it('filters out minimum and maximum values', () => { + expect(mapping.lookup.flat()).not.toContainEqual(1); + expect(mapping.lookup.flat()).not.toContainEqual(10); + }); + }); + + describe('Error scenarios', () => { + const heatmap = { + heatmap: { + name: 'test', + meta: { + type: DataFrameType.HeatmapBuckets, + }, + fields: [ + { + name: 'xMin', + type: FieldType.number, + config: {}, + values: new ArrayVector([1]), + }, + { + name: 'yMin', + type: FieldType.number, + config: {}, + values: new ArrayVector([1, 4, 7]), + }, + { + name: 'count', + type: FieldType.number, + config: {}, + values: new ArrayVector([2, 0, 6]), + }, + ], + length: 3, + }, + xBucketCount: 1, + xBucketSize: 9, + xLayout: BucketLayout.ge, + yBucketCount: 3, + yBucketSize: 3, + yLayout: BucketLayout.ge, + }; + const rawData = { + name: 'origdata', + fields: [ + { + name: 'time', + type: FieldType.time, + config: {}, + values: new ArrayVector([1, 2, 3, 4, 5, 6, 7, 8, 9]), + }, + { + name: 'value', + type: FieldType.number, + config: {}, + values: new ArrayVector([7, 1, 8, 7, 2, 7, 8, 8, 4]), + }, + ], + length: 2, + }; + + it('Will not process heatmap buckets', () => { + expect(() => + getAnnotationMapping( + { + ...heatmap, + heatmap: { + ...heatmap.heatmap, + meta: { + type: DataFrameType.HeatmapBuckets, + }, + }, + }, + rawData + ) + ).toThrow(HEATMAP_NOT_SCANLINES_ERROR); + + expect(() => + getAnnotationMapping( + { + ...heatmap, + heatmap: { + ...heatmap.heatmap, + meta: { + type: DataFrameType.TimeSeriesWide, + }, + }, + }, + rawData + ) + ).toThrow(HEATMAP_NOT_SCANLINES_ERROR); + + expect(() => + getAnnotationMapping( + { + ...heatmap, + heatmap: { + ...heatmap.heatmap, + meta: { + type: undefined, + }, + }, + }, + rawData + ) + ).toThrow(HEATMAP_NOT_SCANLINES_ERROR); + }); + }); +}); diff --git a/public/app/plugins/panel/heatmap-new/fields.ts b/public/app/plugins/panel/heatmap-new/fields.ts index 4d120439e38..145ad8443fe 100644 --- a/public/app/plugins/panel/heatmap-new/fields.ts +++ b/public/app/plugins/panel/heatmap-new/fields.ts @@ -1,12 +1,16 @@ import { DataFrame, DataFrameType, + Field, FieldType, formattedValueToString, getDisplayProcessor, getFieldDisplayName, getValueFormat, GrafanaTheme2, + incrRoundDn, + incrRoundUp, + PanelData, } from '@grafana/data'; import { calculateHeatmapFromData, bucketsToScanlines } from 'app/features/transformers/calculateHeatmap/heatmap'; @@ -17,9 +21,19 @@ export const enum BucketLayout { ge = 'ge', } +export interface HeatmapDataMapping { + lookup: Array; + high: number[]; // index of values bigger than the max Y + low: number[]; // index of values less than the min Y +} + +export const HEATMAP_NOT_SCANLINES_ERROR = 'A calculated heatmap was expected, but not found'; + export interface HeatmapData { // List of heatmap frames heatmap?: DataFrame; + annotations?: DataFrame; + annotationMappings?: HeatmapDataMapping; yAxisValues?: Array; @@ -39,25 +53,25 @@ export interface HeatmapData { warning?: string; } -export function prepareHeatmapData( - frames: DataFrame[] | undefined, - options: PanelOptions, - theme: GrafanaTheme2 -): HeatmapData { +export function prepareHeatmapData(data: PanelData, options: PanelOptions, theme: GrafanaTheme2): HeatmapData { + const frames = data.series; if (!frames?.length) { return {}; } const { source } = options; + + const annotations = data.annotations?.[0]; // TODO: Maybe join on time with frames + if (source === HeatmapSourceMode.Calculate) { // TODO, check for error etc - return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), theme); + return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), annotations, theme); } // Find a well defined heatmap let scanlinesHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapScanlines); if (scanlinesHeatmap) { - return getHeatmapData(scanlinesHeatmap, theme); + return getHeatmapData(scanlinesHeatmap, annotations, theme); } let bucketsHeatmap = frames.find((f) => f.meta?.type === DataFrameType.HeatmapBuckets); @@ -66,19 +80,79 @@ export function prepareHeatmapData( yAxisValues: frames[0].fields.flatMap((field) => field.type === FieldType.number ? getFieldDisplayName(field) : [] ), - ...getHeatmapData(bucketsToScanlines(bucketsHeatmap), theme), + ...getHeatmapData(bucketsToScanlines(bucketsHeatmap), annotations, theme), }; } if (source === HeatmapSourceMode.Data) { - return getHeatmapData(bucketsToScanlines(frames[0]), theme); + return getHeatmapData(bucketsToScanlines(frames[0]), annotations, theme); } // TODO, check for error etc - return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), theme); + return getHeatmapData(calculateHeatmapFromData(frames, options.heatmap ?? {}), annotations, theme); } -const getHeatmapData = (frame: DataFrame, theme: GrafanaTheme2): HeatmapData => { +const getHeatmapFields = (dataFrame: DataFrame): Array => { + const xField: Field | undefined = dataFrame.fields.find((f) => f.name === 'xMin'); + const yField: Field | undefined = dataFrame.fields.find((f) => f.name === 'yMin'); + const countField: Field | undefined = dataFrame.fields.find((f) => f.name === 'count'); + + return [xField, yField, countField]; +}; + +export const getAnnotationMapping = (heatmapData: HeatmapData, rawData: DataFrame): HeatmapDataMapping => { + if (heatmapData.heatmap?.meta?.type !== DataFrameType.HeatmapScanlines) { + throw HEATMAP_NOT_SCANLINES_ERROR; + } + + const [fxs, fys] = getHeatmapFields(heatmapData.heatmap!); + + if (!fxs || !fys) { + throw HEATMAP_NOT_SCANLINES_ERROR; + } + + const mapping: HeatmapDataMapping = { + lookup: new Array(heatmapData.xBucketCount! * heatmapData.yBucketCount!).fill(null), + high: [], + low: [], + }; + + const xos: number[] | undefined = rawData.fields.find((f: Field) => f.type === 'time')?.values.toArray(); + const yos: number[] | undefined = rawData.fields.find((f: Field) => f.type === 'number')?.values.toArray(); + + if (!xos || !yos) { + return mapping; + } + + const xsmin = fxs.values.get(0); + const ysmin = fys.values.get(0); + const xsmax = fxs.values.get(fxs.values.length - 1) + heatmapData.xBucketSize!; + const ysmax = fys.values.get(fys.values.length - 1) + heatmapData.yBucketSize!; + xos.forEach((xo: number, i: number) => { + const yo = yos[i]; + const xBucketIdx = Math.floor(incrRoundDn(incrRoundUp((xo - xsmin) / heatmapData.xBucketSize!, 1e-7), 1e-7)); + const yBucketIdx = Math.floor(incrRoundDn(incrRoundUp((yo - ysmin) / heatmapData.yBucketSize!, 1e-7), 1e-7)); + + if (xo < xsmin || yo < ysmin) { + mapping.low.push(i); + return; + } + + if (xo >= xsmax || yo >= ysmax) { + mapping.high.push(i); + return; + } + + const index = xBucketIdx * heatmapData.yBucketCount! + yBucketIdx; + if (mapping.lookup[index] === null) { + mapping.lookup[index] = []; + } + mapping.lookup[index]?.push(i); + }); + return mapping; +}; + +const getHeatmapData = (frame: DataFrame, annotations: DataFrame | undefined, theme: GrafanaTheme2): HeatmapData => { if (frame.meta?.type !== DataFrameType.HeatmapScanlines) { return { warning: 'Expected heatmap scanlines format', @@ -114,8 +188,9 @@ const getHeatmapData = (frame: DataFrame, theme: GrafanaTheme2): HeatmapData => // The "count" field const disp = frame.fields[2].display ?? getValueFormat('short'); - return { + const data: HeatmapData = { heatmap: frame, + annotations, xBucketSize: xBinIncr, yBucketSize: yBinIncr, xBucketCount: xBinQty, @@ -127,4 +202,10 @@ const getHeatmapData = (frame: DataFrame, theme: GrafanaTheme2): HeatmapData => display: (v) => formattedValueToString(disp(v)), }; + + if (annotations) { + data.annotationMappings = getAnnotationMapping(data, annotations); + } + + return data; }; diff --git a/public/app/plugins/panel/heatmap-new/suggestions.ts b/public/app/plugins/panel/heatmap-new/suggestions.ts index 70d84a51dbc..057e39acf2e 100644 --- a/public/app/plugins/panel/heatmap-new/suggestions.ts +++ b/public/app/plugins/panel/heatmap-new/suggestions.ts @@ -18,7 +18,7 @@ export class HeatmapSuggestionsSupplier { return; } - const info = prepareHeatmapData(builder.data.series, defaultPanelOptions, config.theme2); + const info = prepareHeatmapData(builder.data, defaultPanelOptions, config.theme2); if (!info || info.warning) { return; } diff --git a/public/app/plugins/panel/heatmap-new/utils.test.ts b/public/app/plugins/panel/heatmap-new/utils.test.ts new file mode 100644 index 00000000000..26fecaad53b --- /dev/null +++ b/public/app/plugins/panel/heatmap-new/utils.test.ts @@ -0,0 +1,5 @@ +describe('a test', () => { + it('has to have at least one test', () => { + expect(true).toBeTruthy(); + }); +});