Explore: Add more meta information when line limit is hit (#33069)

* WIP: Add more info ro log line limit, remove redundant info

* Refactor

* Clean up

* Adjust tests

* Adjust spacing

* Add test for new functionality

* Update snapshot

* Change solution, simplify

* Remove redundant variables, makees it more clear

* Update public/app/core/logs_model.ts

Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>

Co-authored-by: Piotr Jamróz <pm.jamroz@gmail.com>

packages/grafana-data/src/datetime/rangeutil.ts

@@ -261,6 +261,23 @@ export function secondsToHms(seconds: number): string {
   return 'less than a millisecond'; //'just now' //or other string you like;
 }
 
+// Format timeSpan (in sec) to string used in log's meta info
+export function msRangeToTimeString(rangeMs: number): string {
+  const rangeSec = Number((rangeMs / 1000).toFixed());
+
+  const h = Math.floor(rangeSec / 60 / 60);
+  const m = Math.floor(rangeSec / 60) - h * 60;
+  const s = Number((rangeSec % 60).toFixed());
+  let formattedH = h ? h + 'h' : '';
+  let formattedM = m ? m + 'min' : '';
+  let formattedS = s ? s + 'sec' : '';
+
+  formattedH && formattedM ? (formattedH = formattedH + ' ') : (formattedH = formattedH);
+  (formattedM || formattedH) && formattedS ? (formattedM = formattedM + ' ') : (formattedM = formattedM);
+
+  return formattedH + formattedM + formattedS || 'less than 1sec';
+}
+
 export function calculateInterval(range: TimeRange, resolution: number, lowLimitInterval?: string): IntervalValues {
   let lowLimitMs = 1; // 1 millisecond default low limit
   if (lowLimitInterval) {

public/app/core/logs_model.test.ts

@@ -8,7 +8,13 @@ import {
   MutableDataFrame,
   toDataFrame,
 } from '@grafana/data';
-import { dataFrameToLogsModel, dedupLogRows, getSeriesProperties, logSeriesToLogsModel } from './logs_model';
+import {
+  dataFrameToLogsModel,
+  dedupLogRows,
+  getSeriesProperties,
+  logSeriesToLogsModel,
+  LIMIT_LABEL,
+} from './logs_model';
 
 describe('dedupLogRows()', () => {
   test('should return rows as is when dedup is set to none', () => {
@@ -246,7 +252,7 @@ describe('dataFrameToLogsModel', () => {
       kind: LogsMetaKind.LabelsMap,
     });
     expect(logsModel.meta![1]).toMatchObject({
-      label: 'Limit',
+      label: LIMIT_LABEL,
       value: `1000 (2 returned)`,
       kind: LogsMetaKind.String,
     });
@@ -316,7 +322,7 @@ describe('dataFrameToLogsModel', () => {
       kind: LogsMetaKind.LabelsMap,
     });
     expect(logsModel.meta![1]).toMatchObject({
-      label: 'Limit',
+      label: LIMIT_LABEL,
       value: `1000 (2 returned)`,
       kind: LogsMetaKind.String,
     });
@@ -554,6 +560,52 @@ describe('dataFrameToLogsModel', () => {
     ]);
   });
 
+  it('should return expected line limit meta info when returned number of series equal the log limit', () => {
+    const series: DataFrame[] = [
+      new MutableDataFrame({
+        fields: [
+          {
+            name: 'time',
+            type: FieldType.time,
+            values: ['2019-04-26T09:28:11.352440161Z', '2019-04-26T14:42:50.991981292Z'],
+          },
+          {
+            name: 'message',
+            type: FieldType.string,
+            values: [
+              't=2019-04-26T11:05:28+0200 lvl=info msg="Initializing DatasourceCacheService" logger=server',
+              't=2019-04-26T16:42:50+0200 lvl=eror msg="new token…t unhashed token=56d9fdc5c8b7400bd51b060eea8ca9d7',
+            ],
+            labels: {
+              filename: '/var/log/grafana/grafana.log',
+              job: 'grafana',
+            },
+          },
+          {
+            name: 'id',
+            type: FieldType.string,
+            values: ['foo', 'bar'],
+          },
+        ],
+        meta: {
+          limit: 2,
+        },
+      }),
+    ];
+    const logsModel = dataFrameToLogsModel(series, 1, 'utc', { from: 1556270591353, to: 1556289770991 });
+    expect(logsModel.meta).toHaveLength(2);
+    expect(logsModel.meta![0]).toMatchObject({
+      label: 'Common labels',
+      value: series[0].fields[1].labels,
+      kind: LogsMetaKind.LabelsMap,
+    });
+    expect(logsModel.meta![1]).toMatchObject({
+      label: LIMIT_LABEL,
+      value: `2 reached, received logs cover 98.44% (5h 14min 40sec) of your selected time range (5h 19min 40sec)`,
+      kind: LogsMetaKind.String,
+    });
+  });
+
   it('should fallback to row index if no id', () => {
     const series: DataFrame[] = [
       toDataFrame({
@@ -588,8 +640,6 @@ describe('logSeriesToLogsModel', () => {
         meta: {
           searchWords: ['test'],
           limit: 1000,
-          stats: [{ displayName: 'Summary: total bytes processed', value: 97048, unit: 'decbytes' }],
-          custom: { lokiQueryStatKey: 'Summary: total bytes processed' },
           preferredVisualisationType: 'logs',
         },
       },
@@ -597,10 +647,7 @@ describe('logSeriesToLogsModel', () => {
 
     const metaData = {
       hasUniqueLabels: false,
-      meta: [
-        { label: 'Limit', value: '1000 (0 returned)', kind: 1 },
-        { label: 'Total bytes processed', value: '97.0  kB', kind: 1 },
-      ],
+      meta: [{ label: LIMIT_LABEL, value: 1000, kind: 0 }],
       rows: [],
     };
 
@@ -634,8 +681,6 @@ describe('logSeriesToLogsModel', () => {
         meta: {
           searchWords: ['test'],
           limit: 1000,
-          stats: [{ displayName: 'Summary: total bytes processed', value: 97048, unit: 'decbytes' }],
-          custom: { lokiQueryStatKey: 'Summary: total bytes processed' },
           preferredVisualisationType: 'logs',
         },
       }),
@@ -646,8 +691,6 @@ describe('logSeriesToLogsModel', () => {
         meta: {
           searchWords: ['test'],
           limit: 1000,
-          stats: [{ displayName: 'Summary: total bytes processed', value: 97048, unit: 'decbytes' }],
-          custom: { lokiQueryStatKey: 'Summary: total bytes processed' },
           preferredVisualisationType: 'logs',
         },
       }),
@@ -656,8 +699,7 @@ describe('logSeriesToLogsModel', () => {
     const logsModel = dataFrameToLogsModel(logSeries, 0, 'utc');
     expect(logsModel.meta).toMatchObject([
       { kind: 2, label: 'Common labels', value: { foo: 'bar', level: 'dbug' } },
-      { kind: 1, label: 'Limit', value: '2000 (3 returned)' },
-      { kind: 1, label: 'Total bytes processed', value: '194  kB' },
+      { kind: 0, label: LIMIT_LABEL, value: 2000 },
     ]);
     expect(logsModel.rows).toHaveLength(3);
     expect(logsModel.rows).toMatchObject([

public/app/core/logs_model.ts

@@ -29,10 +29,11 @@ import {
   dateTime,
   AbsoluteTimeRange,
   sortInAscendingOrder,
+  rangeUtil,
 } from '@grafana/data';
 import { getThemeColor } from 'app/core/utils/colors';
 
-import { SIPrefix } from '@grafana/data/src/valueFormats/symbolFormatters';
+export const LIMIT_LABEL = 'Line limit';
 
 export const LogLevelColor = {
   [LogLevel.critical]: colors[7],
@@ -204,15 +205,21 @@ export function dataFrameToLogsModel(
   const { logSeries } = separateLogsAndMetrics(dataFrame);
   const logsModel = logSeriesToLogsModel(logSeries);
 
-  // unification: Removed logic for using metrics data in LogsModel as with the unification changes this would result
-  // in the incorrect data being used. Instead logs series are always derived from logs.
   if (logsModel) {
     // Create histogram metrics from logs using the interval as bucket size for the line count
     if (intervalMs && logsModel.rows.length > 0) {
       const sortedRows = logsModel.rows.sort(sortInAscendingOrder);
-      const { visibleRange, bucketSize } = getSeriesProperties(sortedRows, intervalMs, absoluteRange);
+      const { visibleRange, bucketSize, visibleRangeMs, requestedRangeMs } = getSeriesProperties(
+        sortedRows,
+        intervalMs,
+        absoluteRange
+      );
       logsModel.visibleRange = visibleRange;
       logsModel.series = makeSeriesForLogs(sortedRows, bucketSize, timeZone);
+
+      if (logsModel.meta) {
+        logsModel.meta = adjustMetaInfo(logsModel, visibleRangeMs, requestedRangeMs);
+      }
     } else {
       logsModel.series = [];
     }
@@ -245,23 +252,31 @@ export function getSeriesProperties(
   let visibleRange = absoluteRange;
   let resolutionIntervalMs = intervalMs;
   let bucketSize = Math.max(resolutionIntervalMs * pxPerBar, minimumBucketSize);
+  let visibleRangeMs;
+  let requestedRangeMs;
   // Clamp time range to visible logs otherwise big parts of the graph might look empty
   if (absoluteRange) {
-    const earliest = sortedRows[0].timeEpochMs;
-    const latest = absoluteRange.to;
-    const visibleRangeMs = latest - earliest;
+    const earliestTsLogs = sortedRows[0].timeEpochMs;
+
+    requestedRangeMs = absoluteRange.to - absoluteRange.from;
+    visibleRangeMs = absoluteRange.to - earliestTsLogs;
+
     if (visibleRangeMs > 0) {
       // Adjust interval bucket size for potentially shorter visible range
-      const clampingFactor = visibleRangeMs / (absoluteRange.to - absoluteRange.from);
+      const clampingFactor = visibleRangeMs / requestedRangeMs;
       resolutionIntervalMs *= clampingFactor;
       // Minimum bucketsize of 1s for nicer graphing
       bucketSize = Math.max(Math.ceil(resolutionIntervalMs * pxPerBar), minimumBucketSize);
       // makeSeriesForLogs() aligns dataspoints with time buckets, so we do the same here to not cut off data
-      const adjustedEarliest = Math.floor(earliest / bucketSize) * bucketSize;
-      visibleRange = { from: adjustedEarliest, to: latest };
+      const adjustedEarliest = Math.floor(earliestTsLogs / bucketSize) * bucketSize;
+      visibleRange = { from: adjustedEarliest, to: absoluteRange.to };
+    } else {
+      // We use visibleRangeMs to calculate range coverage of received logs. However, some data sources are rounding up range in requests. This means that received logs
+      // can (in edge cases) be outside of the requested range and visibleRangeMs < 0. In that case, we want to change visibleRangeMs to be 1 so we can calculate coverage.
+      visibleRangeMs = 1;
     }
   }
-  return { bucketSize, visibleRange };
+  return { bucketSize, visibleRange, visibleRangeMs, requestedRangeMs };
 }
 
 function separateLogsAndMetrics(dataFrames: DataFrame[]) {
@@ -413,26 +428,19 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
       acc[elem.refId] = elem.meta.limit;
       return acc;
     }, {})
-  ).reduce((acc: number, elem: any) => (acc += elem), 0);
+  ).reduce((acc: number, elem: any) => (acc += elem), 0) as number;
 
-  if (limits.length > 0) {
+  if (limitValue > 0) {
     meta.push({
-      label: 'Limit',
-      value: `${limitValue} (${rows.length} returned)`,
-      kind: LogsMetaKind.String,
+      label: LIMIT_LABEL,
+      value: limitValue,
+      kind: LogsMetaKind.Number,
     });
   }
-
-  // Hack to print loki stats in Explore. Should be using proper stats display via drawer in Explore (rework in 7.1)
-  let totalBytes = 0;
-  const queriesVisited: { [refId: string]: boolean } = {};
   // To add just 1 error message
   let errorMetaAdded = false;
 
   for (const series of logSeries) {
-    const totalBytesKey = series.meta?.custom?.lokiQueryStatKey;
-    const { refId } = series; // Stats are per query, keeping track by refId
-
     if (!errorMetaAdded && series.meta?.custom?.error) {
       meta.push({
         label: '',
@@ -441,28 +449,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi
       });
       errorMetaAdded = true;
     }
-
-    if (refId && !queriesVisited[refId]) {
-      if (totalBytesKey && series.meta?.stats) {
-        const byteStat = series.meta.stats.find((stat) => stat.displayName === totalBytesKey);
-        if (byteStat) {
-          totalBytes += byteStat.value;
-        }
-      }
-
-      queriesVisited[refId] = true;
-    }
   }
-
-  if (totalBytes > 0) {
-    const { text, suffix } = SIPrefix('B')(totalBytes);
-    meta.push({
-      label: 'Total bytes processed',
-      value: `${text} ${suffix}`,
-      kind: LogsMetaKind.String,
-    });
-  }
-
   return {
     hasUniqueLabels,
     meta,
@@ -480,3 +467,33 @@ function getIdField(fieldCache: FieldCache): FieldWithIndex | undefined {
   }
   return undefined;
 }
+
+// Used to add additional information to Line limit meta info
+function adjustMetaInfo(logsModel: LogsModel, visibleRangeMs?: number, requestedRangeMs?: number): LogsMetaItem[] {
+  let logsModelMeta = [...logsModel.meta!];
+
+  const limitIndex = logsModelMeta.findIndex((meta) => meta.label === LIMIT_LABEL);
+  const limit = limitIndex && logsModelMeta[limitIndex]?.value;
+
+  if (limit && limit > 0) {
+    let metaLimitValue;
+
+    if (limit === logsModel.rows.length && visibleRangeMs && requestedRangeMs) {
+      const coverage = ((visibleRangeMs / requestedRangeMs) * 100).toFixed(2);
+
+      metaLimitValue = `${limit} reached, received logs cover ${coverage}% (${rangeUtil.msRangeToTimeString(
+        visibleRangeMs
+      )}) of your selected time range (${rangeUtil.msRangeToTimeString(requestedRangeMs)})`;
+    } else {
+      metaLimitValue = `${limit} (${logsModel.rows.length} returned)`;
+    }
+
+    logsModelMeta[limitIndex] = {
+      label: LIMIT_LABEL,
+      value: metaLimitValue,
+      kind: LogsMetaKind.String,
+    };
+  }
+
+  return logsModelMeta;
+}

public/app/features/explore/Logs.tsx

@@ -452,7 +452,7 @@ const getStyles = stylesFactory((theme: GrafanaTheme) => {
       padding: ${theme.spacing.sm} ${theme.spacing.md};
       border-radius: ${theme.border.radius.md};
       margin: ${theme.spacing.md} 0 ${theme.spacing.sm};
-      border: 1px solid ${theme.colors.panelBorder};
+      border: 1px solid ${theme.colors.border2};
     `,
     flipButton: css`
       margin: ${theme.spacing.xs} 0 0 ${theme.spacing.sm};

public/app/features/explore/MetaInfoText.tsx

@@ -10,9 +10,11 @@ const getStyles = stylesFactory((theme: GrafanaTheme) => ({
     margin-bottom: ${theme.spacing.d};
     min-width: 30%;
     display: flex;
+    flex-wrap: wrap;
   `,
   metaItem: css`
     margin-right: ${theme.spacing.d};
+    margin-top: ${theme.spacing.xs};
     display: flex;
     align-items: baseline;
 
@@ -27,6 +29,7 @@ const getStyles = stylesFactory((theme: GrafanaTheme) => ({
   `,
   metaValue: css`
     font-family: ${theme.typography.fontFamily.monospace};
+    font-size: ${theme.typography.size.sm};
   `,
 }));
 

public/app/features/explore/__snapshots__/MetaInfoText.test.tsx.snap

@@ -2,7 +2,7 @@
 
 exports[`MetaInfoText should render component 1`] = `
 <div
-  className="css-1nu680j"
+  className="css-1g6mnuc"
 >
   <Memo(MetaInfoItem)
     key="0-label"