diff --git a/public/app/core/logs_model.ts b/public/app/core/logs_model.ts index 08d53a835d2..09f5bb3a916 100644 --- a/public/app/core/logs_model.ts +++ b/public/app/core/logs_model.ts @@ -95,6 +95,57 @@ export enum LogsDedupStrategy { signature = 'signature', } +export interface LogsParser { + /** + * Value-agnostic matcher for a field label. + * Used to filter rows, and first capture group contains the value. + */ + buildMatcher: (label: string) => RegExp; + /** + * Regex to find a field in the log line. + * First capture group contains the label value, second capture group the value. + */ + fieldRegex: RegExp; + /** + * Function to verify if this is a valid parser for the given line. + * The parser accepts the line unless it returns undefined. + */ + test: (line: string) => any; +} + +export const LogsParsers: { [name: string]: LogsParser } = { + JSON: { + buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"([^"]*)"`), + fieldRegex: /"(\w+)"\s*:\s*"([^"]*)"/, + test: line => { + try { + return JSON.parse(line); + } catch (error) {} + }, + }, + logfmt: { + buildMatcher: label => new RegExp(`(?:^|\\s)${label}=("[^"]*"|\\S+)`), + fieldRegex: /(?:^|\s)(\w+)=("[^"]*"|\S+)/, + test: line => LogsParsers.logfmt.fieldRegex.test(line), + }, +}; + +export function calculateFieldStats(rows: LogRow[], extractor: RegExp): LogsLabelStat[] { + // Consider only rows that satisfy the matcher + const rowsWithField = rows.filter(row => extractor.test(row.entry)); + const rowCount = rowsWithField.length; + + // Get field value counts for eligible rows + const countsByValue = _.countBy(rowsWithField, row => (row as LogRow).entry.match(extractor)[1]); + const sortedCounts = _.chain(countsByValue) + .map((count, value) => ({ count, value, proportion: count / rowCount })) + .sortBy('count') + .reverse() + .value(); + + return sortedCounts; +} + export function calculateLogsLabelStats(rows: LogRow[], label: string): LogsLabelStat[] { // Consider only rows that have the given label const rowsWithLabel = rows.filter(row => row.labels[label] !== undefined); @@ -151,6 +202,19 @@ export function dedupLogRows(logs: LogsModel, strategy: LogsDedupStrategy): Logs }; } +export function getParser(line: string): LogsParser { + let parser; + try { + if (LogsParsers.JSON.test(line)) { + parser = LogsParsers.JSON; + } + } catch (error) {} + if (!parser && LogsParsers.logfmt.test(line)) { + parser = LogsParsers.logfmt; + } + return parser; +} + export function filterLogLevels(logs: LogsModel, hiddenLogLevels: Set): LogsModel { if (hiddenLogLevels.size === 0) { return logs; diff --git a/public/app/core/specs/logs_model.test.ts b/public/app/core/specs/logs_model.test.ts index 22673278b13..85f75b50ed0 100644 --- a/public/app/core/specs/logs_model.test.ts +++ b/public/app/core/specs/logs_model.test.ts @@ -1,4 +1,12 @@ -import { calculateLogsLabelStats, dedupLogRows, LogsDedupStrategy, LogsModel } from '../logs_model'; +import { + calculateFieldStats, + calculateLogsLabelStats, + dedupLogRows, + getParser, + LogsDedupStrategy, + LogsModel, + LogsParsers, +} from '../logs_model'; describe('dedupLogRows()', () => { test('should return rows as is when dedup is set to none', () => { @@ -107,6 +115,50 @@ describe('dedupLogRows()', () => { }); }); +describe('calculateFieldStats()', () => { + test('should return no stats for empty rows', () => { + expect(calculateFieldStats([], /foo=(.*)/)).toEqual([]); + }); + + test('should return no stats if extractor does not match', () => { + const rows = [ + { + entry: 'foo=bar', + }, + ]; + + expect(calculateFieldStats(rows as any, /baz=(.*)/)).toEqual([]); + }); + + test('should return stats for found field', () => { + const rows = [ + { + entry: 'foo="42 + 1"', + }, + { + entry: 'foo=503 baz=foo', + }, + { + entry: 'foo="42 + 1"', + }, + { + entry: 't=2018-12-05T07:44:59+0000 foo=503', + }, + ]; + + expect(calculateFieldStats(rows as any, /foo=("[^"]*"|\S+)/)).toMatchObject([ + { + value: '"42 + 1"', + count: 2, + }, + { + value: '503', + count: 2, + }, + ]); + }); +}); + describe('calculateLogsLabelStats()', () => { test('should return no stats for empty rows', () => { expect(calculateLogsLabelStats([], '')).toEqual([]); @@ -159,3 +211,70 @@ describe('calculateLogsLabelStats()', () => { ]); }); }); + +describe('getParser()', () => { + test('should return no parser on empty line', () => { + expect(getParser('')).toBeUndefined(); + }); + + test('should return no parser on unknown line pattern', () => { + expect(getParser('To Be or not to be')).toBeUndefined(); + }); + + test('should return logfmt parser on key value patterns', () => { + expect(getParser('foo=bar baz="41 + 1')).toEqual(LogsParsers.logfmt); + }); + + test('should return JSON parser on JSON log lines', () => { + // TODO implement other JSON value types than string + expect(getParser('{"foo": "bar", "baz": "41 + 1"}')).toEqual(LogsParsers.JSON); + }); +}); + +describe('LogsParsers', () => { + describe('logfmt', () => { + const parser = LogsParsers.logfmt; + + test('should detect format', () => { + expect(parser.test('foo')).toBeFalsy(); + expect(parser.test('foo=bar')).toBeTruthy(); + }); + + test('should have a valid fieldRegex', () => { + const match = 'foo=bar'.match(parser.fieldRegex); + expect(match).toBeDefined(); + expect(match[1]).toBe('foo'); + expect(match[2]).toBe('bar'); + }); + + test('should build a valid value matcher', () => { + const matcher = parser.buildMatcher('foo'); + const match = 'foo=bar'.match(matcher); + expect(match).toBeDefined(); + expect(match[1]).toBe('bar'); + }); + }); + + describe('JSON', () => { + const parser = LogsParsers.JSON; + + test('should detect format', () => { + expect(parser.test('foo')).toBeFalsy(); + expect(parser.test('{"foo":"bar"}')).toBeTruthy(); + }); + + test('should have a valid fieldRegex', () => { + const match = '{"foo":"bar"}'.match(parser.fieldRegex); + expect(match).toBeDefined(); + expect(match[1]).toBe('foo'); + expect(match[2]).toBe('bar'); + }); + + test('should build a valid value matcher', () => { + const matcher = parser.buildMatcher('foo'); + const match = '{"foo":"bar"}'.match(matcher); + expect(match).toBeDefined(); + expect(match[1]).toBe('bar'); + }); + }); +}); diff --git a/public/app/features/explore/LogLabels.tsx b/public/app/features/explore/LogLabels.tsx index 91e2d44e517..c10ad408a42 100644 --- a/public/app/features/explore/LogLabels.tsx +++ b/public/app/features/explore/LogLabels.tsx @@ -24,7 +24,7 @@ function StatsRow({ active, count, proportion, value }: LogsLabelStat) { } const STATS_ROW_LIMIT = 5; -class Stats extends PureComponent<{ +export class Stats extends PureComponent<{ stats: LogsLabelStat[]; label: string; value: string; @@ -54,7 +54,7 @@ class Stats extends PureComponent<{ {topRows.map(stat => )} - {insertActiveRow && } + {insertActiveRow && activeRow && } {otherCount > 0 && } ); diff --git a/public/app/features/explore/Logs.tsx b/public/app/features/explore/Logs.tsx index 82995390e26..cfc109b1a7d 100644 --- a/public/app/features/explore/Logs.tsx +++ b/public/app/features/explore/Logs.tsx @@ -10,16 +10,20 @@ import { LogsModel, dedupLogRows, filterLogLevels, + getParser, LogLevel, LogsMetaKind, + LogsLabelStat, + LogsParser, LogRow, + calculateFieldStats, } from 'app/core/logs_model'; import { findHighlightChunksInText } from 'app/core/utils/text'; import { Switch } from 'app/core/components/Switch/Switch'; import ToggleButtonGroup, { ToggleButton } from 'app/core/components/ToggleButtonGroup/ToggleButtonGroup'; import Graph from './Graph'; -import LogLabels from './LogLabels'; +import LogLabels, { Stats } from './LogLabels'; const PREVIEW_LIMIT = 100; @@ -38,6 +42,19 @@ const graphOptions = { }, }; +/** + * Renders a highlighted field. + * When hovering, a stats icon is shown. + */ +const FieldHighlight = onClick => props => { + return ( + + {props.children} + onClick(props.children)} /> + + ); +}; + interface RowProps { allRows: LogRow[]; highlighterExpressions?: string[]; @@ -49,57 +66,169 @@ interface RowProps { onClickLabel?: (label: string, value: string) => void; } -function Row({ - allRows, - highlighterExpressions, - onClickLabel, - row, - showDuplicates, - showLabels, - showLocalTime, - showUtc, -}: RowProps) { - const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords); - const highlights = previewHighlights ? highlighterExpressions : row.searchWords; - const needsHighlighter = highlights && highlights.length > 0; - const highlightClassName = classnames('logs-row__match-highlight', { - 'logs-row__match-highlight--preview': previewHighlights, - }); - return ( -
- {showDuplicates && ( -
{row.duplicates > 0 ? `${row.duplicates + 1}x` : null}
- )} -
- {showUtc && ( -
- {row.timestamp} -
- )} - {showLocalTime && ( -
- {row.timeLocal} -
- )} - {showLabels && ( -
- -
- )} -
- {needsHighlighter ? ( - - ) : ( - row.entry +interface RowState { + fieldCount: number; + fieldLabel: string; + fieldStats: LogsLabelStat[]; + fieldValue: string; + parsed: boolean; + parser: LogsParser; + parsedFieldHighlights: string[]; + showFieldStats: boolean; +} + +/** + * Renders a log line. + * + * When user hovers over it for a certain time, it lazily parses the log line. + * Once a parser is found, it will determine fields, that will be highlighted. + * When the user requests stats for a field, they will be calculated and rendered below the row. + */ +class Row extends PureComponent { + mouseMessageTimer: NodeJS.Timer; + + state = { + fieldCount: 0, + fieldLabel: null, + fieldStats: null, + fieldValue: null, + parsed: false, + parser: null, + parsedFieldHighlights: [], + showFieldStats: false, + }; + + componentWillUnmount() { + clearTimeout(this.mouseMessageTimer); + } + + onClickClose = () => { + this.setState({ showFieldStats: false }); + }; + + onClickHighlight = (fieldText: string) => { + const { allRows } = this.props; + const { parser } = this.state; + + const fieldMatch = fieldText.match(parser.fieldRegex); + if (fieldMatch) { + // Build value-agnostic row matcher based on the field label + const fieldLabel = fieldMatch[1]; + const fieldValue = fieldMatch[2]; + const matcher = parser.buildMatcher(fieldLabel); + const fieldStats = calculateFieldStats(allRows, matcher); + const fieldCount = fieldStats.reduce((sum, stat) => sum + stat.count, 0); + + this.setState({ fieldCount, fieldLabel, fieldStats, fieldValue, showFieldStats: true }); + } + }; + + onMouseOverMessage = () => { + // Don't parse right away, user might move along + this.mouseMessageTimer = setTimeout(this.parseMessage, 500); + }; + + onMouseOutMessage = () => { + clearTimeout(this.mouseMessageTimer); + this.setState({ parsed: false }); + }; + + parseMessage = () => { + if (!this.state.parsed) { + const { row } = this.props; + const parser = getParser(row.entry); + if (parser) { + // Use parser to highlight detected fields + const parsedFieldHighlights = []; + this.props.row.entry.replace(new RegExp(parser.fieldRegex, 'g'), substring => { + parsedFieldHighlights.push(substring.trim()); + return ''; + }); + this.setState({ parsedFieldHighlights, parsed: true, parser }); + } + } + }; + + render() { + const { + allRows, + highlighterExpressions, + onClickLabel, + row, + showDuplicates, + showLabels, + showLocalTime, + showUtc, + } = this.props; + const { + fieldCount, + fieldLabel, + fieldStats, + fieldValue, + parsed, + parsedFieldHighlights, + showFieldStats, + } = this.state; + const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords); + const highlights = previewHighlights ? highlighterExpressions : row.searchWords; + const needsHighlighter = highlights && highlights.length > 0; + const highlightClassName = classnames('logs-row__match-highlight', { + 'logs-row__match-highlight--preview': previewHighlights, + }); + return ( +
+ {showDuplicates && ( +
{row.duplicates > 0 ? `${row.duplicates + 1}x` : null}
)} +
+ {showUtc && ( +
+ {row.timestamp} +
+ )} + {showLocalTime && ( +
+ {row.timeLocal} +
+ )} + {showLabels && ( +
+ +
+ )} +
+ {parsed && ( + + )} + {!parsed && + needsHighlighter && ( + + )} + {!parsed && !needsHighlighter && row.entry} + {showFieldStats && ( + + )} +
-
- ); + ); + } } function renderMetaItem(value: any, kind: LogsMetaKind) { diff --git a/public/sass/components/_panel_logs.scss b/public/sass/components/_panel_logs.scss index 6f1c43cfe1d..572e6e890d7 100644 --- a/public/sass/components/_panel_logs.scss +++ b/public/sass/components/_panel_logs.scss @@ -158,6 +158,28 @@ $column-horizontal-spacing: 10px; text-align: right; } +.logs-row__field-highlight { + // Undoing mark styling + background: inherit; + padding: inherit; + border-bottom: 1px dotted $typeahead-selected-color; + + .logs-row__field-highlight--icon { + margin-left: 0.5em; + cursor: pointer; + display: none; + } +} + +.logs-row__field-highlight:hover { + color: $typeahead-selected-color; + border-bottom-style: solid; + + .logs-row__field-highlight--icon { + display: inline; + } +} + .logs-label { display: inline-block; padding: 0 2px;