Explore: Align multiple log volumes (#64356)

* Align log volumes on x an y axes

* Move helper functions to logs/utils

* Add tests

* Simplify supplementaryQueries.ts

* Fix tests

* Revert code simplifications

To simplify the PR, this can be added in a separate PR

* Fix reusing logs volume when limited/non-limited are used

* Use more specific property name

* Add missing property

* Stretch graph to selected range but only if there's data available

* Fix unit tests

* Fix calculating maximum when bars are stacked

* Sort log volumes by data source name

* Simplify logic to determine if log volumes can be zoomed in

packages/grafana-data/src/types/logs.ts

@@ -205,29 +205,6 @@ export type LogsVolumeCustomMetaData = {
   sourceQuery: DataQuery;
 };
 
-export const getLogsVolumeAbsoluteRange = (
-  dataFrames: DataFrame[],
-  defaultRange: AbsoluteTimeRange
-): AbsoluteTimeRange => {
-  return dataFrames[0].meta?.custom?.absoluteRange || defaultRange;
-};
-
-export const getLogsVolumeDataSourceInfo = (dataFrames: DataFrame[]): { name: string } | null => {
-  const customMeta = dataFrames[0]?.meta?.custom;
-
-  if (customMeta && customMeta.datasourceName) {
-    return {
-      name: customMeta.datasourceName,
-    };
-  }
-
-  return null;
-};
-
-export const isLogsVolumeLimited = (dataFrames: DataFrame[]) => {
-  return dataFrames[0]?.meta?.custom?.logsVolumeType === LogsVolumeType.Limited;
-};
-
 /**
  * Data sources that support supplementary queries in Explore.
  * This will enable users to see additional data when running original queries.

public/app/core/logsModel.test.ts

@@ -13,6 +13,7 @@ import {
   LogRowModel,
   LogsDedupStrategy,
   LogsMetaKind,
+  LogsVolumeCustomMetaData,
   LogsVolumeType,
   MutableDataFrame,
   sortDataFrame,
@@ -1207,6 +1208,16 @@ describe('logs volume', () => {
   it('applies correct meta data', async () => {
     setup(setupMultipleResults);
 
+    const logVolumeCustomMeta: LogsVolumeCustomMetaData = {
+      sourceQuery: { refId: 'A', target: 'volume query 1' } as DataQuery,
+      datasourceName: 'loki',
+      logsVolumeType: LogsVolumeType.FullRange,
+      absoluteRange: {
+        from: FROM.valueOf(),
+        to: TO.valueOf(),
+      },
+    };
+
     await expect(volumeProvider).toEmitValuesWith((received) => {
       expect(received).toContainEqual({ state: LoadingState.Loading, error: undefined, data: [] });
       expect(received).toContainEqual({
@@ -1216,15 +1227,7 @@ describe('logs volume', () => {
           expect.objectContaining({
             fields: expect.anything(),
             meta: {
-              custom: {
+              custom: logVolumeCustomMeta,
-                sourceQuery: { refId: 'A', target: 'volume query 1' },
-                datasourceName: 'loki',
-                logsVolumeType: LogsVolumeType.FullRange,
-                absoluteRange: {
-                  from: FROM.valueOf(),
-                  to: TO.valueOf(),
-                },
-              },
             },
           }),
           expect.anything(),
@@ -1236,6 +1239,16 @@ describe('logs volume', () => {
   it('applies correct meta data when streaming', async () => {
     setup(setupMultipleResultsStreaming);
 
+    const logVolumeCustomMeta: LogsVolumeCustomMetaData = {
+      sourceQuery: { refId: 'A', target: 'volume query 1' } as DataQuery,
+      datasourceName: 'loki',
+      logsVolumeType: LogsVolumeType.FullRange,
+      absoluteRange: {
+        from: FROM.valueOf(),
+        to: TO.valueOf(),
+      },
+    };
+
     await expect(volumeProvider).toEmitValuesWith((received) => {
       expect(received).toContainEqual({ state: LoadingState.Loading, error: undefined, data: [] });
       expect(received).toContainEqual({
@@ -1245,15 +1258,7 @@ describe('logs volume', () => {
           expect.objectContaining({
             fields: expect.anything(),
             meta: {
-              custom: {
+              custom: logVolumeCustomMeta,
-                sourceQuery: { refId: 'A', target: 'volume query 1' },
-                datasourceName: 'loki',
-                logsVolumeType: LogsVolumeType.FullRange,
-                absoluteRange: {
-                  from: FROM.valueOf(),
-                  to: TO.valueOf(),
-                },
-              },
             },
           }),
           expect.anything(),

public/app/features/explore/Graph/ExploreGraph.tsx

@@ -54,6 +54,7 @@ interface Props {
   onChangeTime: (timeRange: AbsoluteTimeRange) => void;
   graphStyle: ExploreGraphStyle;
   anchorToZero?: boolean;
+  yAxisMaximum?: number;
   eventBus: EventBus;
 }
 
@@ -71,6 +72,7 @@ export function ExploreGraph({
   graphStyle,
   tooltipDisplayMode = TooltipDisplayMode.Single,
   anchorToZero = false,
+  yAxisMaximum,
   eventBus,
 }: Props) {
   const theme = useTheme2();
@@ -94,6 +96,7 @@ export function ExploreGraph({
   const [fieldConfig, setFieldConfig] = useState<FieldConfigSource>({
     defaults: {
       min: anchorToZero ? 0 : undefined,
+      max: yAxisMaximum || undefined,
       color: {
         mode: FieldColorModeId.PaletteClassic,
       },
@@ -106,7 +109,10 @@ export function ExploreGraph({
     overrides: [],
   });
 
-  const styledFieldConfig = useMemo(() => applyGraphStyle(fieldConfig, graphStyle), [fieldConfig, graphStyle]);
+  const styledFieldConfig = useMemo(
+    () => applyGraphStyle(fieldConfig, graphStyle, yAxisMaximum),
+    [fieldConfig, graphStyle, yAxisMaximum]
+  );
 
   const dataWithConfig = useMemo(() => {
     return applyFieldOverrides({

public/app/features/explore/Graph/exploreGraphStyleUtils.ts

@@ -6,12 +6,14 @@ import { ExploreGraphStyle } from 'app/types';
 
 export type FieldConfig = FieldConfigSource<GraphFieldConfig>;
 
-export function applyGraphStyle(config: FieldConfig, style: ExploreGraphStyle): FieldConfig {
+export function applyGraphStyle(config: FieldConfig, style: ExploreGraphStyle, maximum?: number): FieldConfig {
   return produce(config, (draft) => {
     if (draft.defaults.custom === undefined) {
       draft.defaults.custom = {};
     }
 
+    draft.defaults.max = maximum;
+
     const { custom } = draft.defaults;
 
     if (custom.stacking === undefined) {

public/app/features/explore/LogsVolumePanel.test.tsx

@@ -24,6 +24,7 @@ function renderPanel(logsVolumeData?: DataQueryResponse) {
       onLoadLogsVolume={() => {}}
       onHiddenSeriesChanged={() => null}
       eventBus={new EventBusSrv()}
+      allLogsVolumeMaximum={20}
     />
   );
 }

public/app/features/explore/LogsVolumePanel.tsx

@@ -9,17 +9,17 @@ import {
   SplitOpen,
   TimeZone,
   EventBus,
-  isLogsVolumeLimited,
-  getLogsVolumeAbsoluteRange,
   GrafanaTheme2,
-  getLogsVolumeDataSourceInfo,
 } from '@grafana/data';
 import { Icon, Tooltip, TooltipDisplayMode, useStyles2, useTheme2 } from '@grafana/ui';
 
+import { getLogsVolumeDataSourceInfo, isLogsVolumeLimited } from '../logs/utils';
+
 import { ExploreGraph } from './Graph/ExploreGraph';
 
 type Props = {
   logsVolumeData: DataQueryResponse | undefined;
+  allLogsVolumeMaximum: number;
   absoluteRange: AbsoluteTimeRange;
   timeZone: TimeZone;
   splitOpen: SplitOpen;
@@ -31,7 +31,7 @@ type Props = {
 };
 
 export function LogsVolumePanel(props: Props) {
-  const { width, timeZone, splitOpen, onUpdateTimeRange, onHiddenSeriesChanged } = props;
+  const { width, timeZone, splitOpen, onUpdateTimeRange, onHiddenSeriesChanged, allLogsVolumeMaximum } = props;
   const theme = useTheme2();
   const styles = useStyles2(getStyles);
   const spacing = parseInt(theme.spacing(2).slice(0, -2), 10);
@@ -55,10 +55,6 @@ export function LogsVolumePanel(props: Props) {
       .join('. ');
   }
 
-  const range = isLogsVolumeLimited(logsVolumeData.data)
-    ? getLogsVolumeAbsoluteRange(logsVolumeData.data, props.absoluteRange)
-    : props.absoluteRange;
-
   let LogsVolumePanelContent;
 
   if (logsVolumeData?.data) {
@@ -70,13 +66,14 @@ export function LogsVolumePanel(props: Props) {
           data={logsVolumeData.data}
           height={height}
           width={width - spacing * 2}
-          absoluteRange={range}
+          absoluteRange={props.absoluteRange}
           onChangeTime={onUpdateTimeRange}
           timeZone={timeZone}
           splitOpenFn={splitOpen}
           tooltipDisplayMode={TooltipDisplayMode.Multi}
           onHiddenSeriesChanged={onHiddenSeriesChanged}
           anchorToZero
+          yAxisMaximum={allLogsVolumeMaximum}
           eventBus={props.eventBus}
         />
       );

public/app/features/explore/LogsVolumePanelList.tsx

@@ -1,5 +1,5 @@
 import { css } from '@emotion/css';
-import { groupBy, mapValues } from 'lodash';
+import { flatten, groupBy, mapValues, sortBy } from 'lodash';
 import React, { useMemo } from 'react';
 
 import {
@@ -8,14 +8,13 @@ import {
   DataQueryResponse,
   EventBus,
   GrafanaTheme2,
-  isLogsVolumeLimited,
   LoadingState,
   SplitOpen,
   TimeZone,
 } from '@grafana/data';
 import { Button, InlineField, useStyles2 } from '@grafana/ui';
 
-import { mergeLogsVolumeDataFrames } from '../logs/utils';
+import { mergeLogsVolumeDataFrames, isLogsVolumeLimited, getLogsVolumeMaximumRange } from '../logs/utils';
 
 import { LogsVolumePanel } from './LogsVolumePanel';
 import { SupplementaryResultError } from './SupplementaryResultError';
@@ -46,11 +45,25 @@ export const LogsVolumePanelList = ({
   timeZone,
   onClose,
 }: Props) => {
-  const logVolumes: Record<string, DataFrame[]> = useMemo(() => {
+  const {
-    const grouped = groupBy(logsVolumeData?.data || [], 'meta.custom.datasourceName');
+    logVolumes,
-    return mapValues(grouped, (value) => {
+    maximumValue: allLogsVolumeMaximumValue,
-      return mergeLogsVolumeDataFrames(value);
+    maximumRange: allLogsVolumeMaximumRange,
+  } = useMemo(() => {
+    let maximumValue = -Infinity;
+    const sorted = sortBy(logsVolumeData?.data || [], 'meta.custom.datasourceName');
+    const grouped = groupBy(sorted, 'meta.custom.datasourceName');
+    const logVolumes = mapValues(grouped, (value) => {
+      const mergedData = mergeLogsVolumeDataFrames(value);
+      maximumValue = Math.max(maximumValue, mergedData.maximum);
+      return mergedData.dataFrames;
     });
+    const maximumRange = getLogsVolumeMaximumRange(flatten(Object.values(logVolumes)));
+    return {
+      maximumValue,
+      maximumRange,
+      logVolumes,
+    };
   }, [logsVolumeData]);
 
   const styles = useStyles2(getStyles);
@@ -64,6 +77,11 @@ export const LogsVolumePanelList = ({
 
   const timeoutError = isTimeoutErrorResponse(logsVolumeData);
 
+  const visibleRange = {
+    from: Math.max(absoluteRange.from, allLogsVolumeMaximumRange.from),
+    to: Math.min(absoluteRange.to, allLogsVolumeMaximumRange.to),
+  };
+
   if (logsVolumeData?.state === LoadingState.Loading) {
     return <span>Loading...</span>;
   } else if (timeoutError) {
@@ -87,7 +105,8 @@ export const LogsVolumePanelList = ({
         return (
           <LogsVolumePanel
             key={index}
-            absoluteRange={absoluteRange}
+            absoluteRange={visibleRange}
+            allLogsVolumeMaximum={allLogsVolumeMaximumValue}
             width={width}
             logsVolumeData={logsVolumeData}
             onUpdateTimeRange={onUpdateTimeRange}

public/app/features/explore/state/query.ts

@@ -15,6 +15,7 @@ import {
   hasQueryImportSupport,
   HistoryItem,
   LoadingState,
+  LogsVolumeType,
   PanelEvents,
   QueryFixAction,
   SupplementaryQueryType,
@@ -695,6 +696,12 @@ function canReuseSupplementaryQueryData(
     head
   );
 
+  const allSupportZoomingIn = supplementaryQueryData.data.every((data: DataFrame) => {
+    // If log volume is based on returned log lines (i.e. LogsVolumeType.Limited),
+    // zooming in may return different results, so we don't want to reuse the data
+    return data.meta?.custom?.logsVolumeType === LogsVolumeType.FullRange;
+  });
+
   const allQueriesAreTheSame = deepEqual(newQueriesByRefId, existingDataByRefId);
 
   const allResultsHaveWiderRange = supplementaryQueryData.data.every((data: DataFrame) => {
@@ -707,7 +714,7 @@ function canReuseSupplementaryQueryData(
     return hasWiderRange;
   });
 
-  return allQueriesAreTheSame && allResultsHaveWiderRange;
+  return allSupportZoomingIn && allQueriesAreTheSame && allResultsHaveWiderRange;
 }
 
 /**

public/app/features/logs/utils.test.ts

@@ -1,6 +1,6 @@
 import {
+  AbsoluteTimeRange,
   ArrayVector,
-  DataFrame,
   FieldType,
   Labels,
   LogLevel,
@@ -8,6 +8,7 @@ import {
   LogsModel,
   LogsSortOrder,
   MutableDataFrame,
+  DataFrame,
 } from '@grafana/data';
 
 import {
@@ -16,6 +17,7 @@ import {
   checkLogsError,
   getLogLevel,
   getLogLevelFromKey,
+  getLogsVolumeMaximumRange,
   logRowsToReadableJson,
   mergeLogsVolumeDataFrames,
   sortLogsResult,
@@ -296,13 +298,15 @@ describe('mergeLogsVolumeDataFrames', () => {
     const debugVolume1 = mockLogVolume('debug', [2, 3], [2, 3]);
     const debugVolume2 = mockLogVolume('debug', [1, 5], [1, 0]);
 
-    // error 1:    - - - - - 1
+    // error 1:    1 - - - - 1
-    // error 2:    1 - - - - 1
+    // error 2:    1 - - - - -
-    // total:      1 - - - - 2
+    // total:      2 - - - - 1
     const errorVolume1 = mockLogVolume('error', [1, 6], [1, 1]);
     const errorVolume2 = mockLogVolume('error', [1], [1]);
 
-    const merged = mergeLogsVolumeDataFrames([
+    // all totals: 6 5 4 - 0 2
+
+    const { dataFrames: merged, maximum } = mergeLogsVolumeDataFrames([
       infoVolume1,
       infoVolume2,
       debugVolume1,
@@ -365,5 +369,40 @@ describe('mergeLogsVolumeDataFrames', () => {
         ],
       },
     ]);
+    expect(maximum).toBe(6);
+  });
+});
+
+describe('getLogsVolumeDimensions', () => {
+  function mockLogVolumeDataFrame(values: number[], absoluteRange: AbsoluteTimeRange) {
+    return new MutableDataFrame({
+      meta: {
+        custom: {
+          absoluteRange,
+        },
+      },
+      fields: [
+        {
+          name: 'time',
+          type: FieldType.time,
+          values: new ArrayVector([]),
+        },
+        {
+          name: 'value',
+          type: FieldType.number,
+          values: new ArrayVector(values),
+        },
+      ],
+    });
+  }
+
+  it('calculates the maximum value and range of all log volumes', () => {
+    const maximumRange = getLogsVolumeMaximumRange([
+      mockLogVolumeDataFrame([], { from: 5, to: 20 }),
+      mockLogVolumeDataFrame([], { from: 10, to: 25 }),
+      mockLogVolumeDataFrame([], { from: 7, to: 23 }),
+    ]);
+
+    expect(maximumRange).toEqual({ from: 5, to: 25 });
   });
 });

public/app/features/logs/utils.ts

@@ -12,6 +12,7 @@ import {
   FieldType,
   MutableDataFrame,
   QueryResultMeta,
+  LogsVolumeType,
 } from '@grafana/data';
 
 import { getDataframeFields } from './components/logParser';
@@ -163,12 +164,37 @@ export function logRowsToReadableJson(logs: LogRowModel[]) {
   });
 }
 
-export const mergeLogsVolumeDataFrames = (dataFrames: DataFrame[]): DataFrame[] => {
+export const getLogsVolumeMaximumRange = (dataFrames: DataFrame[]) => {
+  let widestRange = { from: Infinity, to: -Infinity };
+
+  dataFrames.forEach((dataFrame: DataFrame) => {
+    const meta = dataFrame.meta?.custom || {};
+    if (meta.absoluteRange?.from && meta.absoluteRange?.to) {
+      widestRange = {
+        from: Math.min(widestRange.from, meta.absoluteRange.from),
+        to: Math.max(widestRange.to, meta.absoluteRange.to),
+      };
+    }
+  });
+
+  return widestRange;
+};
+
+/**
+ * Merge data frames by level and calculate maximum total value for all levels together
+ */
+export const mergeLogsVolumeDataFrames = (dataFrames: DataFrame[]): { dataFrames: DataFrame[]; maximum: number } => {
   if (dataFrames.length === 0) {
     throw new Error('Cannot aggregate data frames: there must be at least one data frame to aggregate');
   }
 
+  // aggregate by level (to produce data frames)
   const aggregated: Record<string, Record<number, number>> = {};
+
+  // aggregate totals to align Y axis when multiple log volumes are shown
+  const totals: Record<number, number> = {};
+  let maximumValue = -Infinity;
+
   const configs: Record<
     string,
     { meta?: QueryResultMeta; valueFieldConfig: FieldConfig; timeFieldConfig: FieldConfig }
@@ -201,6 +227,9 @@ export const mergeLogsVolumeDataFrames = (dataFrames: DataFrame[]): DataFrame[]
       const value: number = valueField.values.get(pointIndex);
       aggregated[level] ??= {};
       aggregated[level][time] = (aggregated[level][time] || 0) + value;
+
+      totals[time] = (totals[time] || 0) + value;
+      maximumValue = Math.max(totals[time], maximumValue);
     }
   });
 
@@ -225,5 +254,21 @@ export const mergeLogsVolumeDataFrames = (dataFrames: DataFrame[]): DataFrame[]
     results.push(levelDataFrame);
   });
 
-  return results;
+  return { dataFrames: results, maximum: maximumValue };
+};
+
+export const getLogsVolumeDataSourceInfo = (dataFrames: DataFrame[]): { name: string } | null => {
+  const customMeta = dataFrames[0]?.meta?.custom;
+
+  if (customMeta && customMeta.datasourceName) {
+    return {
+      name: customMeta.datasourceName,
+    };
+  }
+
+  return null;
+};
+
+export const isLogsVolumeLimited = (dataFrames: DataFrame[]) => {
+  return dataFrames[0]?.meta?.custom?.logsVolumeType === LogsVolumeType.Limited;
 };