From 0490dbb88d1217c6d21eff17a0719ba7c3177b53 Mon Sep 17 00:00:00 2001 From: Andrej Ocenas Date: Mon, 30 Sep 2019 14:44:15 +0200 Subject: [PATCH] Explore: Generate log row uid (#18994) --- .../grafana-data/src/dataframe/FieldCache.ts | 2 +- packages/grafana-data/src/types/logs.ts | 1 + .../Logs/LogRowContextProvider.test.ts | 2 + .../src/components/Logs/LogRows.tsx | 4 +- public/app/core/logs_model.ts | 14 ++++ public/app/core/specs/logs_model.test.ts | 30 ++++++++- public/app/core/utils/explore.test.ts | 2 + public/app/features/explore/LiveLogs.tsx | 34 ++-------- .../explore/utils/ResultProcessor.test.ts | 3 + .../app/plugins/datasource/loki/datasource.ts | 10 +++ .../datasource/loki/live_streams.test.ts | 1 + .../plugins/datasource/loki/live_streams.ts | 1 + .../loki/result_transformer.test.ts | 67 +++++++++++++------ .../datasource/loki/result_transformer.ts | 16 +++-- 14 files changed, 129 insertions(+), 58 deletions(-) diff --git a/packages/grafana-data/src/dataframe/FieldCache.ts b/packages/grafana-data/src/dataframe/FieldCache.ts index 07f6941e21d..e7dc574d70d 100644 --- a/packages/grafana-data/src/dataframe/FieldCache.ts +++ b/packages/grafana-data/src/dataframe/FieldCache.ts @@ -1,6 +1,6 @@ import { Field, DataFrame, FieldType, guessFieldTypeForField } from '../index'; -interface FieldWithIndex extends Field { +export interface FieldWithIndex extends Field { index: number; } diff --git a/packages/grafana-data/src/types/logs.ts b/packages/grafana-data/src/types/logs.ts index 524588a5403..d02811a5228 100644 --- a/packages/grafana-data/src/types/logs.ts +++ b/packages/grafana-data/src/types/logs.ts @@ -48,6 +48,7 @@ export interface LogRowModel { timeEpochMs: number; timeLocal: string; timeUtc: string; + uid: string; uniqueLabels?: Labels; } diff --git a/packages/grafana-ui/src/components/Logs/LogRowContextProvider.test.ts b/packages/grafana-ui/src/components/Logs/LogRowContextProvider.test.ts index de46c188fe7..b853eb31778 100644 --- a/packages/grafana-ui/src/components/Logs/LogRowContextProvider.test.ts +++ b/packages/grafana-ui/src/components/Logs/LogRowContextProvider.test.ts @@ -33,6 +33,7 @@ describe('getRowContexts', () => { timeLocal: '', timeUtc: '', timestamp: '4', + uid: '1', }; let called = false; @@ -65,6 +66,7 @@ describe('getRowContexts', () => { timeLocal: '', timeUtc: '', timestamp: '4', + uid: '1', }; let called = false; diff --git a/packages/grafana-ui/src/components/Logs/LogRows.tsx b/packages/grafana-ui/src/components/Logs/LogRows.tsx index 81bc082d412..b2f60ca0cec 100644 --- a/packages/grafana-ui/src/components/Logs/LogRows.tsx +++ b/packages/grafana-ui/src/components/Logs/LogRows.tsx @@ -105,7 +105,7 @@ class UnThemedLogRows extends PureComponent { !deferLogs && // Only inject highlighterExpression in the first set for performance reasons firstRows.map((row, index) => ( { renderAll && lastRows.map((row, index) => ( { 't=2019-04-26T16:42:50+0200 lvl=eror msg="new token…t unhashed token=56d9fdc5c8b7400bd51b060eea8ca9d7', ], }, + { + name: 'id', + type: FieldType.string, + values: ['foo', 'bar'], + }, ], meta: { limit: 1000, @@ -233,6 +238,7 @@ describe('dataFrameToLogsModel', () => { labels: { filename: '/var/log/grafana/grafana.log', job: 'grafana' }, logLevel: 'info', uniqueLabels: {}, + uid: 'foo', }, { timestamp: '2019-04-26T14:42:50.991981292Z', @@ -240,6 +246,7 @@ describe('dataFrameToLogsModel', () => { labels: { filename: '/var/log/grafana/grafana.log', job: 'grafana' }, logLevel: 'error', uniqueLabels: {}, + uid: 'bar', }, ]); @@ -367,7 +374,6 @@ describe('dataFrameToLogsModel', () => { kind: LogsMetaKind.LabelsMap, }); }); - // it('given multiple series with equal times should return expected logs model', () => { const series: DataFrame[] = [ toDataFrame({ @@ -459,4 +465,26 @@ describe('dataFrameToLogsModel', () => { }, ]); }); + + it('should fallback to row index if no id', () => { + const series: DataFrame[] = [ + toDataFrame({ + labels: { foo: 'bar' }, + fields: [ + { + name: 'ts', + type: FieldType.time, + values: ['1970-01-01T00:00:00Z'], + }, + { + name: 'line', + type: FieldType.string, + values: ['WARN boooo 1'], + }, + ], + }), + ]; + const logsModel = dataFrameToLogsModel(series, 0); + expect(logsModel.rows[0].uid).toBe('0'); + }); }); diff --git a/public/app/core/utils/explore.test.ts b/public/app/core/utils/explore.test.ts index 336335fd8a3..2fa2811c030 100644 --- a/public/app/core/utils/explore.test.ts +++ b/public/app/core/utils/explore.test.ts @@ -383,6 +383,7 @@ describe('sortLogsResult', () => { timeFromNow: '', timeLocal: '', timeUtc: '', + uid: '1', }; const sameAsFirstRow = firstRow; const secondRow = { @@ -396,6 +397,7 @@ describe('sortLogsResult', () => { timeFromNow: '', timeLocal: '', timeUtc: '', + uid: '2', }; describe('when called with SortOrder.Descending', () => { diff --git a/public/app/features/explore/LiveLogs.tsx b/public/app/features/explore/LiveLogs.tsx index 2bafb9694a0..078dd97c14e 100644 --- a/public/app/features/explore/LiveLogs.tsx +++ b/public/app/features/explore/LiveLogs.tsx @@ -1,7 +1,6 @@ import React, { PureComponent } from 'react'; import { css, cx } from 'emotion'; import tinycolor from 'tinycolor2'; -import { last } from 'lodash'; import { Themeable, withTheme, GrafanaTheme, getLogRowStyles } from '@grafana/ui'; import { LogsModel, LogRowModel, TimeZone } from '@grafana/data'; @@ -21,7 +20,7 @@ const getStyles = (theme: GrafanaTheme) => ({ margin-top: auto !important; } `, - logsRowFresh: css` + logsRowFade: css` label: logs-row-fresh; color: ${theme.colors.text}; background-color: ${tinycolor(theme.colors.blueLight) @@ -39,9 +38,6 @@ const getStyles = (theme: GrafanaTheme) => ({ } } `, - logsRowOld: css` - label: logs-row-old; - `, logsRowsIndicator: css` font-size: ${theme.typography.size.md}; padding-top: ${theme.spacing.sm}; @@ -64,7 +60,6 @@ export interface Props extends Themeable { interface State { logsResultToRender?: LogsModel; - lastTimestamp: number; } class LiveLogs extends PureComponent { @@ -76,7 +71,6 @@ class LiveLogs extends PureComponent { super(props); this.state = { logsResultToRender: props.logsResult, - lastTimestamp: 0, }; } @@ -106,10 +100,6 @@ class LiveLogs extends PureComponent { // our state, but we do not show the updates, this allows us start again showing correct result after resuming // without creating a gap in the log results. logsResultToRender: nextProps.logsResult, - lastTimestamp: - state.logsResultToRender && last(state.logsResultToRender.rows) - ? last(state.logsResultToRender.rows).timeEpochMs - : 0, }; } else { return null; @@ -141,15 +131,6 @@ class LiveLogs extends PureComponent { return rowsToRender; }; - /** - * Check if row is fresh so we can apply special styling. This is bit naive and does not take into account rows - * which arrive out of order. Because loki datasource sends full data instead of deltas we need to compare the - * data and this is easier than doing some intersection of some uuid of each row (which we do not have now anyway) - */ - isFresh = (row: LogRowModel): boolean => { - return row.timeEpochMs > this.state.lastTimestamp; - }; - render() { const { theme, timeZone, onPause, onResume, isPaused } = this.props; const styles = getStyles(theme); @@ -163,23 +144,20 @@ class LiveLogs extends PureComponent { className={cx(['logs-rows', styles.logsRowsLive])} ref={this.scrollContainerRef} > - {this.rowsToRender().map((row: LogRowModel, index) => { + {this.rowsToRender().map((row: LogRowModel) => { return ( -
+
{showUtc && ( -
+
{row.timeUtc}
)} {!showUtc && ( -
+
{row.timeLocal}
)} -
{row.entry}
+
{row.entry}
); })} diff --git a/public/app/features/explore/utils/ResultProcessor.test.ts b/public/app/features/explore/utils/ResultProcessor.test.ts index efe8ee84eca..8679c6ce51f 100644 --- a/public/app/features/explore/utils/ResultProcessor.test.ts +++ b/public/app/features/explore/utils/ResultProcessor.test.ts @@ -156,6 +156,7 @@ describe('ResultProcessor', () => { timeLocal: 'format() jest mocked', timeUtc: 'format() jest mocked', timestamp: 300, + uid: '2', uniqueLabels: {}, }, { @@ -170,6 +171,7 @@ describe('ResultProcessor', () => { timeLocal: 'format() jest mocked', timeUtc: 'format() jest mocked', timestamp: 200, + uid: '1', uniqueLabels: {}, }, { @@ -184,6 +186,7 @@ describe('ResultProcessor', () => { timeLocal: 'format() jest mocked', timeUtc: 'format() jest mocked', timestamp: 100, + uid: '0', uniqueLabels: {}, }, ], diff --git a/public/app/plugins/datasource/loki/datasource.ts b/public/app/plugins/datasource/loki/datasource.ts index f04c5601761..46289639044 100644 --- a/public/app/plugins/datasource/loki/datasource.ts +++ b/public/app/plugins/datasource/loki/datasource.ts @@ -168,6 +168,16 @@ export class LokiDatasource extends DataSourceApi { return series; }; + /** + * Runs live queries which in this case means creating a websocket and listening on it for new logs. + * This returns a bit different dataFrame than runQueries as it returns single dataframe even if there are multiple + * Loki streams, sets only common labels on dataframe.labels and has additional dataframe.fields.labels for unique + * labels per row. + * + * @param options + * @param observer Callback that will be called with new data. Is optional but only because we run this function + * even if there are no live targets defined in the options which would mean this is noop and observer is not called. + */ runLiveQuery = (options: DataQueryRequest, target: LokiQuery): Observable => { const liveTarget = this.prepareLiveTarget(target, options); const stream = this.streams.getStream(liveTarget); diff --git a/public/app/plugins/datasource/loki/live_streams.test.ts b/public/app/plugins/datasource/loki/live_streams.test.ts index e1e6f3f030c..d153ed8dca2 100644 --- a/public/app/plugins/datasource/loki/live_streams.test.ts +++ b/public/app/plugins/datasource/loki/live_streams.test.ts @@ -50,6 +50,7 @@ describe('Live Stream Tests', () => { const last = { ...view.get(view.length - 1) }; expect(last).toEqual({ ts: '2019-08-28T20:50:40.118944705Z', + id: '2019-08-28T20:50:40.118944705Z_{filename="/var/log/sntpc.log", job="varlogs"}', line: 'Kittens', labels: { filename: '/var/log/sntpc.log' }, }); diff --git a/public/app/plugins/datasource/loki/live_streams.ts b/public/app/plugins/datasource/loki/live_streams.ts index b23ae1be129..9c84284ecd9 100644 --- a/public/app/plugins/datasource/loki/live_streams.ts +++ b/public/app/plugins/datasource/loki/live_streams.ts @@ -31,6 +31,7 @@ export class LiveStreams { data.addField({ name: 'ts', type: FieldType.time, config: { title: 'Time' } }); data.addField({ name: 'line', type: FieldType.string }); data.addField({ name: 'labels', type: FieldType.other }); + data.addField({ name: 'id', type: FieldType.string }); stream = webSocket(target.url).pipe( finalize(() => { diff --git a/public/app/plugins/datasource/loki/result_transformer.test.ts b/public/app/plugins/datasource/loki/result_transformer.test.ts index 797c2d21377..9cbd62d8eb0 100644 --- a/public/app/plugins/datasource/loki/result_transformer.test.ts +++ b/public/app/plugins/datasource/loki/result_transformer.test.ts @@ -1,26 +1,29 @@ -import { logStreamToDataFrame } from './result_transformer'; +import { logStreamToDataFrame, appendResponseToBufferedData } from './result_transformer'; +import { FieldType, MutableDataFrame } from '@grafana/data'; +import { LokiLogsStream } from './types'; -describe('convert loki response to DataFrame', () => { - const streams = [ - { - labels: '{foo="bar"}', - entries: [ - { - line: "foo: 'bar'", - ts: '1970-01-01T00:00:00Z', - }, - ], - }, - { - labels: '{bar="foo"}', - entries: [ - { - line: "bar: 'foo'", - ts: '1970-01-01T00:00:00Z', - }, - ], - }, - ]; +const streams: LokiLogsStream[] = [ + { + labels: '{foo="bar"}', + entries: [ + { + line: "foo: 'bar'", + ts: '1970-01-01T00:00:00Z', + }, + ], + }, + { + labels: '{bar="foo"}', + entries: [ + { + line: "bar: 'foo'", + ts: '1970-01-01T00:00:00Z', + }, + ], + }, +]; + +describe('logStreamToDataFrame', () => { it('converts streams to series', () => { const data = streams.map(stream => logStreamToDataFrame(stream)); @@ -28,7 +31,27 @@ describe('convert loki response to DataFrame', () => { expect(data[0].labels['foo']).toEqual('bar'); expect(data[0].fields[0].values.get(0)).toEqual(streams[0].entries[0].ts); expect(data[0].fields[1].values.get(0)).toEqual(streams[0].entries[0].line); + expect(data[0].fields[2].values.get(0)).toEqual('1970-01-01T00:00:00Z_{foo="bar"}'); expect(data[1].fields[0].values.get(0)).toEqual(streams[1].entries[0].ts); expect(data[1].fields[1].values.get(0)).toEqual(streams[1].entries[0].line); + expect(data[1].fields[2].values.get(0)).toEqual('1970-01-01T00:00:00Z_{bar="foo"}'); + }); +}); + +describe('appendResponseToBufferedData', () => { + it('appends response', () => { + const data = new MutableDataFrame(); + data.addField({ name: 'ts', type: FieldType.time, config: { title: 'Time' } }); + data.addField({ name: 'line', type: FieldType.string }); + data.addField({ name: 'labels', type: FieldType.other }); + data.addField({ name: 'id', type: FieldType.string }); + + appendResponseToBufferedData({ streams }, data); + expect(data.get(0)).toEqual({ + ts: '1970-01-01T00:00:00Z', + line: "foo: 'bar'", + labels: { foo: 'bar' }, + id: '1970-01-01T00:00:00Z_{foo="bar"}', + }); }); }); diff --git a/public/app/plugins/datasource/loki/result_transformer.ts b/public/app/plugins/datasource/loki/result_transformer.ts index c6f672302c2..209c64836b1 100644 --- a/public/app/plugins/datasource/loki/result_transformer.ts +++ b/public/app/plugins/datasource/loki/result_transformer.ts @@ -19,10 +19,13 @@ export function logStreamToDataFrame(stream: LokiLogsStream, reverse?: boolean, } const times = new ArrayVector([]); const lines = new ArrayVector([]); + const uids = new ArrayVector([]); for (const entry of stream.entries) { - times.add(entry.ts || entry.timestamp); + const ts = entry.ts || entry.timestamp; + times.add(ts); lines.add(entry.line); + uids.add(`${ts}_${stream.labels}`); } if (reverse) { @@ -36,6 +39,7 @@ export function logStreamToDataFrame(stream: LokiLogsStream, reverse?: boolean, fields: [ { name: 'ts', type: FieldType.time, config: { title: 'Time' }, values: times }, // Time { name: 'line', type: FieldType.string, config: {}, values: lines }, // Line + { name: 'id', type: FieldType.string, config: {}, values: uids }, ], length: times.length, }; @@ -45,22 +49,26 @@ export function logStreamToDataFrame(stream: LokiLogsStream, reverse?: boolean, * Transform LokiResponse data and appends it to MutableDataFrame. Used for streaming where the dataFrame can be * a CircularDataFrame creating a fixed size rolling buffer. * TODO: Probably could be unified with the logStreamToDataFrame function. + * @param response + * @param data Needs to have ts, line, labels, id as fields */ export function appendResponseToBufferedData(response: LokiResponse, data: MutableDataFrame) { - // Should we do anythign with: response.dropped_entries? + // Should we do anything with: response.dropped_entries? const streams: LokiLogsStream[] = response.streams; if (streams && streams.length) { for (const stream of streams) { // Find unique labels const labels = parseLabels(stream.labels); - const unique = findUniqueLabels(labels, data.labels); + const unique = findUniqueLabels(labels, data.labels || {}); // Add each line for (const entry of stream.entries) { - data.values.ts.add(entry.ts || entry.timestamp); + const ts = entry.ts || entry.timestamp; + data.values.ts.add(ts); data.values.line.add(entry.line); data.values.labels.add(unique); + data.values.id.add(`${ts}_${stream.labels}`); } } }