diff --git a/packages/grafana-data/src/datetime/rangeutil.ts b/packages/grafana-data/src/datetime/rangeutil.ts index c2201e435ed..bc2c01a8c0d 100644 --- a/packages/grafana-data/src/datetime/rangeutil.ts +++ b/packages/grafana-data/src/datetime/rangeutil.ts @@ -261,6 +261,23 @@ export function secondsToHms(seconds: number): string { return 'less than a millisecond'; //'just now' //or other string you like; } +// Format timeSpan (in sec) to string used in log's meta info +export function msRangeToTimeString(rangeMs: number): string { + const rangeSec = Number((rangeMs / 1000).toFixed()); + + const h = Math.floor(rangeSec / 60 / 60); + const m = Math.floor(rangeSec / 60) - h * 60; + const s = Number((rangeSec % 60).toFixed()); + let formattedH = h ? h + 'h' : ''; + let formattedM = m ? m + 'min' : ''; + let formattedS = s ? s + 'sec' : ''; + + formattedH && formattedM ? (formattedH = formattedH + ' ') : (formattedH = formattedH); + (formattedM || formattedH) && formattedS ? (formattedM = formattedM + ' ') : (formattedM = formattedM); + + return formattedH + formattedM + formattedS || 'less than 1sec'; +} + export function calculateInterval(range: TimeRange, resolution: number, lowLimitInterval?: string): IntervalValues { let lowLimitMs = 1; // 1 millisecond default low limit if (lowLimitInterval) { diff --git a/public/app/core/logs_model.test.ts b/public/app/core/logs_model.test.ts index 5f0f02c2b80..22df3385430 100644 --- a/public/app/core/logs_model.test.ts +++ b/public/app/core/logs_model.test.ts @@ -8,7 +8,13 @@ import { MutableDataFrame, toDataFrame, } from '@grafana/data'; -import { dataFrameToLogsModel, dedupLogRows, getSeriesProperties, logSeriesToLogsModel } from './logs_model'; +import { + dataFrameToLogsModel, + dedupLogRows, + getSeriesProperties, + logSeriesToLogsModel, + LIMIT_LABEL, +} from './logs_model'; describe('dedupLogRows()', () => { test('should return rows as is when dedup is set to none', () => { @@ -246,7 +252,7 @@ describe('dataFrameToLogsModel', () => { kind: LogsMetaKind.LabelsMap, }); expect(logsModel.meta![1]).toMatchObject({ - label: 'Limit', + label: LIMIT_LABEL, value: `1000 (2 returned)`, kind: LogsMetaKind.String, }); @@ -316,7 +322,7 @@ describe('dataFrameToLogsModel', () => { kind: LogsMetaKind.LabelsMap, }); expect(logsModel.meta![1]).toMatchObject({ - label: 'Limit', + label: LIMIT_LABEL, value: `1000 (2 returned)`, kind: LogsMetaKind.String, }); @@ -554,6 +560,52 @@ describe('dataFrameToLogsModel', () => { ]); }); + it('should return expected line limit meta info when returned number of series equal the log limit', () => { + const series: DataFrame[] = [ + new MutableDataFrame({ + fields: [ + { + name: 'time', + type: FieldType.time, + values: ['2019-04-26T09:28:11.352440161Z', '2019-04-26T14:42:50.991981292Z'], + }, + { + name: 'message', + type: FieldType.string, + values: [ + 't=2019-04-26T11:05:28+0200 lvl=info msg="Initializing DatasourceCacheService" logger=server', + 't=2019-04-26T16:42:50+0200 lvl=eror msg="new token…t unhashed token=56d9fdc5c8b7400bd51b060eea8ca9d7', + ], + labels: { + filename: '/var/log/grafana/grafana.log', + job: 'grafana', + }, + }, + { + name: 'id', + type: FieldType.string, + values: ['foo', 'bar'], + }, + ], + meta: { + limit: 2, + }, + }), + ]; + const logsModel = dataFrameToLogsModel(series, 1, 'utc', { from: 1556270591353, to: 1556289770991 }); + expect(logsModel.meta).toHaveLength(2); + expect(logsModel.meta![0]).toMatchObject({ + label: 'Common labels', + value: series[0].fields[1].labels, + kind: LogsMetaKind.LabelsMap, + }); + expect(logsModel.meta![1]).toMatchObject({ + label: LIMIT_LABEL, + value: `2 reached, received logs cover 98.44% (5h 14min 40sec) of your selected time range (5h 19min 40sec)`, + kind: LogsMetaKind.String, + }); + }); + it('should fallback to row index if no id', () => { const series: DataFrame[] = [ toDataFrame({ @@ -588,8 +640,6 @@ describe('logSeriesToLogsModel', () => { meta: { searchWords: ['test'], limit: 1000, - stats: [{ displayName: 'Summary: total bytes processed', value: 97048, unit: 'decbytes' }], - custom: { lokiQueryStatKey: 'Summary: total bytes processed' }, preferredVisualisationType: 'logs', }, }, @@ -597,10 +647,7 @@ describe('logSeriesToLogsModel', () => { const metaData = { hasUniqueLabels: false, - meta: [ - { label: 'Limit', value: '1000 (0 returned)', kind: 1 }, - { label: 'Total bytes processed', value: '97.0 kB', kind: 1 }, - ], + meta: [{ label: LIMIT_LABEL, value: 1000, kind: 0 }], rows: [], }; @@ -634,8 +681,6 @@ describe('logSeriesToLogsModel', () => { meta: { searchWords: ['test'], limit: 1000, - stats: [{ displayName: 'Summary: total bytes processed', value: 97048, unit: 'decbytes' }], - custom: { lokiQueryStatKey: 'Summary: total bytes processed' }, preferredVisualisationType: 'logs', }, }), @@ -646,8 +691,6 @@ describe('logSeriesToLogsModel', () => { meta: { searchWords: ['test'], limit: 1000, - stats: [{ displayName: 'Summary: total bytes processed', value: 97048, unit: 'decbytes' }], - custom: { lokiQueryStatKey: 'Summary: total bytes processed' }, preferredVisualisationType: 'logs', }, }), @@ -656,8 +699,7 @@ describe('logSeriesToLogsModel', () => { const logsModel = dataFrameToLogsModel(logSeries, 0, 'utc'); expect(logsModel.meta).toMatchObject([ { kind: 2, label: 'Common labels', value: { foo: 'bar', level: 'dbug' } }, - { kind: 1, label: 'Limit', value: '2000 (3 returned)' }, - { kind: 1, label: 'Total bytes processed', value: '194 kB' }, + { kind: 0, label: LIMIT_LABEL, value: 2000 }, ]); expect(logsModel.rows).toHaveLength(3); expect(logsModel.rows).toMatchObject([ diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index 1d03167c9bf..62317195323 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -29,10 +29,11 @@ import { dateTime, AbsoluteTimeRange, sortInAscendingOrder, + rangeUtil, } from '@grafana/data'; import { getThemeColor } from 'app/core/utils/colors'; -import { SIPrefix } from '@grafana/data/src/valueFormats/symbolFormatters'; +export const LIMIT_LABEL = 'Line limit'; export const LogLevelColor = { [LogLevel.critical]: colors[7], @@ -204,15 +205,21 @@ export function dataFrameToLogsModel( const { logSeries } = separateLogsAndMetrics(dataFrame); const logsModel = logSeriesToLogsModel(logSeries); - // unification: Removed logic for using metrics data in LogsModel as with the unification changes this would result - // in the incorrect data being used. Instead logs series are always derived from logs. if (logsModel) { // Create histogram metrics from logs using the interval as bucket size for the line count if (intervalMs && logsModel.rows.length > 0) { const sortedRows = logsModel.rows.sort(sortInAscendingOrder); - const { visibleRange, bucketSize } = getSeriesProperties(sortedRows, intervalMs, absoluteRange); + const { visibleRange, bucketSize, visibleRangeMs, requestedRangeMs } = getSeriesProperties( + sortedRows, + intervalMs, + absoluteRange + ); logsModel.visibleRange = visibleRange; logsModel.series = makeSeriesForLogs(sortedRows, bucketSize, timeZone); + + if (logsModel.meta) { + logsModel.meta = adjustMetaInfo(logsModel, visibleRangeMs, requestedRangeMs); + } } else { logsModel.series = []; } @@ -245,23 +252,31 @@ export function getSeriesProperties( let visibleRange = absoluteRange; let resolutionIntervalMs = intervalMs; let bucketSize = Math.max(resolutionIntervalMs * pxPerBar, minimumBucketSize); + let visibleRangeMs; + let requestedRangeMs; // Clamp time range to visible logs otherwise big parts of the graph might look empty if (absoluteRange) { - const earliest = sortedRows[0].timeEpochMs; - const latest = absoluteRange.to; - const visibleRangeMs = latest - earliest; + const earliestTsLogs = sortedRows[0].timeEpochMs; + + requestedRangeMs = absoluteRange.to - absoluteRange.from; + visibleRangeMs = absoluteRange.to - earliestTsLogs; + if (visibleRangeMs > 0) { // Adjust interval bucket size for potentially shorter visible range - const clampingFactor = visibleRangeMs / (absoluteRange.to - absoluteRange.from); + const clampingFactor = visibleRangeMs / requestedRangeMs; resolutionIntervalMs *= clampingFactor; // Minimum bucketsize of 1s for nicer graphing bucketSize = Math.max(Math.ceil(resolutionIntervalMs * pxPerBar), minimumBucketSize); // makeSeriesForLogs() aligns dataspoints with time buckets, so we do the same here to not cut off data - const adjustedEarliest = Math.floor(earliest / bucketSize) * bucketSize; - visibleRange = { from: adjustedEarliest, to: latest }; + const adjustedEarliest = Math.floor(earliestTsLogs / bucketSize) * bucketSize; + visibleRange = { from: adjustedEarliest, to: absoluteRange.to }; + } else { + // We use visibleRangeMs to calculate range coverage of received logs. However, some data sources are rounding up range in requests. This means that received logs + // can (in edge cases) be outside of the requested range and visibleRangeMs < 0. In that case, we want to change visibleRangeMs to be 1 so we can calculate coverage. + visibleRangeMs = 1; } } - return { bucketSize, visibleRange }; + return { bucketSize, visibleRange, visibleRangeMs, requestedRangeMs }; } function separateLogsAndMetrics(dataFrames: DataFrame[]) { @@ -413,26 +428,19 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi acc[elem.refId] = elem.meta.limit; return acc; }, {}) - ).reduce((acc: number, elem: any) => (acc += elem), 0); + ).reduce((acc: number, elem: any) => (acc += elem), 0) as number; - if (limits.length > 0) { + if (limitValue > 0) { meta.push({ - label: 'Limit', - value: `${limitValue} (${rows.length} returned)`, - kind: LogsMetaKind.String, + label: LIMIT_LABEL, + value: limitValue, + kind: LogsMetaKind.Number, }); } - - // Hack to print loki stats in Explore. Should be using proper stats display via drawer in Explore (rework in 7.1) - let totalBytes = 0; - const queriesVisited: { [refId: string]: boolean } = {}; // To add just 1 error message let errorMetaAdded = false; for (const series of logSeries) { - const totalBytesKey = series.meta?.custom?.lokiQueryStatKey; - const { refId } = series; // Stats are per query, keeping track by refId - if (!errorMetaAdded && series.meta?.custom?.error) { meta.push({ label: '', @@ -441,28 +449,7 @@ export function logSeriesToLogsModel(logSeries: DataFrame[]): LogsModel | undefi }); errorMetaAdded = true; } - - if (refId && !queriesVisited[refId]) { - if (totalBytesKey && series.meta?.stats) { - const byteStat = series.meta.stats.find((stat) => stat.displayName === totalBytesKey); - if (byteStat) { - totalBytes += byteStat.value; - } - } - - queriesVisited[refId] = true; - } } - - if (totalBytes > 0) { - const { text, suffix } = SIPrefix('B')(totalBytes); - meta.push({ - label: 'Total bytes processed', - value: `${text} ${suffix}`, - kind: LogsMetaKind.String, - }); - } - return { hasUniqueLabels, meta, @@ -480,3 +467,33 @@ function getIdField(fieldCache: FieldCache): FieldWithIndex | undefined { } return undefined; } + +// Used to add additional information to Line limit meta info +function adjustMetaInfo(logsModel: LogsModel, visibleRangeMs?: number, requestedRangeMs?: number): LogsMetaItem[] { + let logsModelMeta = [...logsModel.meta!]; + + const limitIndex = logsModelMeta.findIndex((meta) => meta.label === LIMIT_LABEL); + const limit = limitIndex && logsModelMeta[limitIndex]?.value; + + if (limit && limit > 0) { + let metaLimitValue; + + if (limit === logsModel.rows.length && visibleRangeMs && requestedRangeMs) { + const coverage = ((visibleRangeMs / requestedRangeMs) * 100).toFixed(2); + + metaLimitValue = `${limit} reached, received logs cover ${coverage}% (${rangeUtil.msRangeToTimeString( + visibleRangeMs + )}) of your selected time range (${rangeUtil.msRangeToTimeString(requestedRangeMs)})`; + } else { + metaLimitValue = `${limit} (${logsModel.rows.length} returned)`; + } + + logsModelMeta[limitIndex] = { + label: LIMIT_LABEL, + value: metaLimitValue, + kind: LogsMetaKind.String, + }; + } + + return logsModelMeta; +} diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index e22d63eb1a7..9f16e3f6e95 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -452,7 +452,7 @@ const getStyles = stylesFactory((theme: GrafanaTheme) => { padding: ${theme.spacing.sm} ${theme.spacing.md}; border-radius: ${theme.border.radius.md}; margin: ${theme.spacing.md} 0 ${theme.spacing.sm}; - border: 1px solid ${theme.colors.panelBorder}; + border: 1px solid ${theme.colors.border2}; `, flipButton: css` margin: ${theme.spacing.xs} 0 0 ${theme.spacing.sm}; diff --git a/public/app/features/explore/MetaInfoText.tsx b/public/app/features/explore/MetaInfoText.tsx index 8f7cdb1566f..640fed4ff3a 100644 --- a/public/app/features/explore/MetaInfoText.tsx +++ b/public/app/features/explore/MetaInfoText.tsx @@ -10,9 +10,11 @@ const getStyles = stylesFactory((theme: GrafanaTheme) => ({ margin-bottom: ${theme.spacing.d}; min-width: 30%; display: flex; + flex-wrap: wrap; `, metaItem: css` margin-right: ${theme.spacing.d}; + margin-top: ${theme.spacing.xs}; display: flex; align-items: baseline; @@ -27,6 +29,7 @@ const getStyles = stylesFactory((theme: GrafanaTheme) => ({ `, metaValue: css` font-family: ${theme.typography.fontFamily.monospace}; + font-size: ${theme.typography.size.sm}; `, })); diff --git a/public/app/features/explore/__snapshots__/MetaInfoText.test.tsx.snap b/public/app/features/explore/__snapshots__/MetaInfoText.test.tsx.snap index 9d15cf5f6b3..8da60de5152 100644 --- a/public/app/features/explore/__snapshots__/MetaInfoText.test.tsx.snap +++ b/public/app/features/explore/__snapshots__/MetaInfoText.test.tsx.snap @@ -2,7 +2,7 @@ exports[`MetaInfoText should render component 1`] = `