Explore: Logging line parsing and field stats

Lazily parse lines and display stats for field when clicked on.

- log line parsers for JSON (basic), and logfmt
- delayed parsing in case user moves to other line
- reuse label stats for field stats

public/app/core/logs_model.ts

@@ -95,6 +95,57 @@ export enum LogsDedupStrategy {
   signature = 'signature',
 }
 
+export interface LogsParser {
+  /**
+   * Value-agnostic matcher for a field label.
+   * Used to filter rows, and first capture group contains the value.
+   */
+  buildMatcher: (label: string) => RegExp;
+  /**
+   * Regex to find a field in the log line.
+   * First capture group contains the label value, second capture group the value.
+   */
+  fieldRegex: RegExp;
+  /**
+   * Function to verify if this is a valid parser for the given line.
+   * The parser accepts the line unless it returns undefined.
+   */
+  test: (line: string) => any;
+}
+
+export const LogsParsers: { [name: string]: LogsParser } = {
+  JSON: {
+    buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"([^"]*)"`),
+    fieldRegex: /"(\w+)"\s*:\s*"([^"]*)"/,
+    test: line => {
+      try {
+        return JSON.parse(line);
+      } catch (error) {}
+    },
+  },
+  logfmt: {
+    buildMatcher: label => new RegExp(`(?:^|\\s)${label}=("[^"]*"|\\S+)`),
+    fieldRegex: /(?:^|\s)(\w+)=("[^"]*"|\S+)/,
+    test: line => LogsParsers.logfmt.fieldRegex.test(line),
+  },
+};
+
+export function calculateFieldStats(rows: LogRow[], extractor: RegExp): LogsLabelStat[] {
+  // Consider only rows that satisfy the matcher
+  const rowsWithField = rows.filter(row => extractor.test(row.entry));
+  const rowCount = rowsWithField.length;
+
+  // Get field value counts for eligible rows
+  const countsByValue = _.countBy(rowsWithField, row => (row as LogRow).entry.match(extractor)[1]);
+  const sortedCounts = _.chain(countsByValue)
+    .map((count, value) => ({ count, value, proportion: count / rowCount }))
+    .sortBy('count')
+    .reverse()
+    .value();
+
+  return sortedCounts;
+}
+
 export function calculateLogsLabelStats(rows: LogRow[], label: string): LogsLabelStat[] {
   // Consider only rows that have the given label
   const rowsWithLabel = rows.filter(row => row.labels[label] !== undefined);
@@ -151,6 +202,19 @@ export function dedupLogRows(logs: LogsModel, strategy: LogsDedupStrategy): Logs
   };
 }
 
+export function getParser(line: string): LogsParser {
+  let parser;
+  try {
+    if (LogsParsers.JSON.test(line)) {
+      parser = LogsParsers.JSON;
+    }
+  } catch (error) {}
+  if (!parser && LogsParsers.logfmt.test(line)) {
+    parser = LogsParsers.logfmt;
+  }
+  return parser;
+}
+
 export function filterLogLevels(logs: LogsModel, hiddenLogLevels: Set<LogLevel>): LogsModel {
   if (hiddenLogLevels.size === 0) {
     return logs;

public/app/core/specs/logs_model.test.ts

@@ -1,4 +1,12 @@
-import { calculateLogsLabelStats, dedupLogRows, LogsDedupStrategy, LogsModel } from '../logs_model';
+import {
+  calculateFieldStats,
+  calculateLogsLabelStats,
+  dedupLogRows,
+  getParser,
+  LogsDedupStrategy,
+  LogsModel,
+  LogsParsers,
+} from '../logs_model';
 
 describe('dedupLogRows()', () => {
   test('should return rows as is when dedup is set to none', () => {
@@ -107,6 +115,50 @@ describe('dedupLogRows()', () => {
   });
 });
 
+describe('calculateFieldStats()', () => {
+  test('should return no stats for empty rows', () => {
+    expect(calculateFieldStats([], /foo=(.*)/)).toEqual([]);
+  });
+
+  test('should return no stats if extractor does not match', () => {
+    const rows = [
+      {
+        entry: 'foo=bar',
+      },
+    ];
+
+    expect(calculateFieldStats(rows as any, /baz=(.*)/)).toEqual([]);
+  });
+
+  test('should return stats for found field', () => {
+    const rows = [
+      {
+        entry: 'foo="42 + 1"',
+      },
+      {
+        entry: 'foo=503 baz=foo',
+      },
+      {
+        entry: 'foo="42 + 1"',
+      },
+      {
+        entry: 't=2018-12-05T07:44:59+0000 foo=503',
+      },
+    ];
+
+    expect(calculateFieldStats(rows as any, /foo=("[^"]*"|\S+)/)).toMatchObject([
+      {
+        value: '"42 + 1"',
+        count: 2,
+      },
+      {
+        value: '503',
+        count: 2,
+      },
+    ]);
+  });
+});
+
 describe('calculateLogsLabelStats()', () => {
   test('should return no stats for empty rows', () => {
     expect(calculateLogsLabelStats([], '')).toEqual([]);
@@ -159,3 +211,70 @@ describe('calculateLogsLabelStats()', () => {
     ]);
   });
 });
+
+describe('getParser()', () => {
+  test('should return no parser on empty line', () => {
+    expect(getParser('')).toBeUndefined();
+  });
+
+  test('should return no parser on unknown line pattern', () => {
+    expect(getParser('To Be or not to be')).toBeUndefined();
+  });
+
+  test('should return logfmt parser on key value patterns', () => {
+    expect(getParser('foo=bar baz="41 + 1')).toEqual(LogsParsers.logfmt);
+  });
+
+  test('should return JSON parser on JSON log lines', () => {
+    // TODO implement other JSON value types than string
+    expect(getParser('{"foo": "bar", "baz": "41 + 1"}')).toEqual(LogsParsers.JSON);
+  });
+});
+
+describe('LogsParsers', () => {
+  describe('logfmt', () => {
+    const parser = LogsParsers.logfmt;
+
+    test('should detect format', () => {
+      expect(parser.test('foo')).toBeFalsy();
+      expect(parser.test('foo=bar')).toBeTruthy();
+    });
+
+    test('should have a valid fieldRegex', () => {
+      const match = 'foo=bar'.match(parser.fieldRegex);
+      expect(match).toBeDefined();
+      expect(match[1]).toBe('foo');
+      expect(match[2]).toBe('bar');
+    });
+
+    test('should build a valid value matcher', () => {
+      const matcher = parser.buildMatcher('foo');
+      const match = 'foo=bar'.match(matcher);
+      expect(match).toBeDefined();
+      expect(match[1]).toBe('bar');
+    });
+  });
+
+  describe('JSON', () => {
+    const parser = LogsParsers.JSON;
+
+    test('should detect format', () => {
+      expect(parser.test('foo')).toBeFalsy();
+      expect(parser.test('{"foo":"bar"}')).toBeTruthy();
+    });
+
+    test('should have a valid fieldRegex', () => {
+      const match = '{"foo":"bar"}'.match(parser.fieldRegex);
+      expect(match).toBeDefined();
+      expect(match[1]).toBe('foo');
+      expect(match[2]).toBe('bar');
+    });
+
+    test('should build a valid value matcher', () => {
+      const matcher = parser.buildMatcher('foo');
+      const match = '{"foo":"bar"}'.match(matcher);
+      expect(match).toBeDefined();
+      expect(match[1]).toBe('bar');
+    });
+  });
+});

public/app/features/explore/LogLabels.tsx

@@ -24,7 +24,7 @@ function StatsRow({ active, count, proportion, value }: LogsLabelStat) {
 }
 
 const STATS_ROW_LIMIT = 5;
-class Stats extends PureComponent<{
+export class Stats extends PureComponent<{
   stats: LogsLabelStat[];
   label: string;
   value: string;
@@ -54,7 +54,7 @@ class Stats extends PureComponent<{
           <span className="logs-stats__icon fa fa-window-close" onClick={onClickClose} />
         </div>
         {topRows.map(stat => <StatsRow key={stat.value} {...stat} active={stat.value === value} />)}
-        {insertActiveRow && <StatsRow key={activeRow.value} {...activeRow} active />}
+        {insertActiveRow && activeRow && <StatsRow key={activeRow.value} {...activeRow} active />}
         {otherCount > 0 && <StatsRow key="__OTHERS__" count={otherCount} value="Other" proportion={otherProportion} />}
       </>
     );

public/app/features/explore/Logs.tsx

@@ -10,16 +10,20 @@ import {
   LogsModel,
   dedupLogRows,
   filterLogLevels,
+  getParser,
   LogLevel,
   LogsMetaKind,
+  LogsLabelStat,
+  LogsParser,
   LogRow,
+  calculateFieldStats,
 } from 'app/core/logs_model';
 import { findHighlightChunksInText } from 'app/core/utils/text';
 import { Switch } from 'app/core/components/Switch/Switch';
 import ToggleButtonGroup, { ToggleButton } from 'app/core/components/ToggleButtonGroup/ToggleButtonGroup';
 
 import Graph from './Graph';
-import LogLabels from './LogLabels';
+import LogLabels, { Stats } from './LogLabels';
 
 const PREVIEW_LIMIT = 100;
 
@@ -38,6 +42,19 @@ const graphOptions = {
   },
 };
 
+/**
+ * Renders a highlighted field.
+ * When hovering, a stats icon is shown.
+ */
+const FieldHighlight = onClick => props => {
+  return (
+    <span className={props.className} style={props.style}>
+      {props.children}
+      <span className="logs-row__field-highlight--icon fa fa-signal" onClick={() => onClick(props.children)} />
+    </span>
+  );
+};
+
 interface RowProps {
   allRows: LogRow[];
   highlighterExpressions?: string[];
@@ -49,57 +66,169 @@ interface RowProps {
   onClickLabel?: (label: string, value: string) => void;
 }
 
-function Row({
+interface RowState {
-  allRows,
+  fieldCount: number;
-  highlighterExpressions,
+  fieldLabel: string;
-  onClickLabel,
+  fieldStats: LogsLabelStat[];
-  row,
+  fieldValue: string;
-  showDuplicates,
+  parsed: boolean;
-  showLabels,
+  parser: LogsParser;
-  showLocalTime,
+  parsedFieldHighlights: string[];
-  showUtc,
+  showFieldStats: boolean;
-}: RowProps) {
+}
-  const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords);
+
-  const highlights = previewHighlights ? highlighterExpressions : row.searchWords;
+/**
-  const needsHighlighter = highlights && highlights.length > 0;
+ * Renders a log line.
-  const highlightClassName = classnames('logs-row__match-highlight', {
+ *
-    'logs-row__match-highlight--preview': previewHighlights,
+ * When user hovers over it for a certain time, it lazily parses the log line.
-  });
+ * Once a parser is found, it will determine fields, that will be highlighted.
-  return (
+ * When the user requests stats for a field, they will be calculated and rendered below the row.
-    <div className="logs-row">
+ */
-      {showDuplicates && (
+class Row extends PureComponent<RowProps, RowState> {
-        <div className="logs-row__duplicates">{row.duplicates > 0 ? `${row.duplicates + 1}x` : null}</div>
+  mouseMessageTimer: NodeJS.Timer;
-      )}
+
-      <div className={row.logLevel ? `logs-row__level logs-row__level--${row.logLevel}` : ''} />
+  state = {
-      {showUtc && (
+    fieldCount: 0,
-        <div className="logs-row__time" title={`Local: ${row.timeLocal} (${row.timeFromNow})`}>
+    fieldLabel: null,
-          {row.timestamp}
+    fieldStats: null,
-        </div>
+    fieldValue: null,
-      )}
+    parsed: false,
-      {showLocalTime && (
+    parser: null,
-        <div className="logs-row__time" title={`${row.timestamp} (${row.timeFromNow})`}>
+    parsedFieldHighlights: [],
-          {row.timeLocal}
+    showFieldStats: false,
-        </div>
+  };
-      )}
+
-      {showLabels && (
+  componentWillUnmount() {
-        <div className="logs-row__labels">
+    clearTimeout(this.mouseMessageTimer);
-          <LogLabels allRows={allRows} labels={row.uniqueLabels} onClickLabel={onClickLabel} />
+  }
-        </div>
+
-      )}
+  onClickClose = () => {
-      <div className="logs-row__message">
+    this.setState({ showFieldStats: false });
-        {needsHighlighter ? (
+  };
-          <Highlighter
+
-            textToHighlight={row.entry}
+  onClickHighlight = (fieldText: string) => {
-            searchWords={highlights}
+    const { allRows } = this.props;
-            findChunks={findHighlightChunksInText}
+    const { parser } = this.state;
-            highlightClassName={highlightClassName}
+
-          />
+    const fieldMatch = fieldText.match(parser.fieldRegex);
-        ) : (
+    if (fieldMatch) {
-          row.entry
+      // Build value-agnostic row matcher based on the field label
+      const fieldLabel = fieldMatch[1];
+      const fieldValue = fieldMatch[2];
+      const matcher = parser.buildMatcher(fieldLabel);
+      const fieldStats = calculateFieldStats(allRows, matcher);
+      const fieldCount = fieldStats.reduce((sum, stat) => sum + stat.count, 0);
+
+      this.setState({ fieldCount, fieldLabel, fieldStats, fieldValue, showFieldStats: true });
+    }
+  };
+
+  onMouseOverMessage = () => {
+    // Don't parse right away, user might move along
+    this.mouseMessageTimer = setTimeout(this.parseMessage, 500);
+  };
+
+  onMouseOutMessage = () => {
+    clearTimeout(this.mouseMessageTimer);
+    this.setState({ parsed: false });
+  };
+
+  parseMessage = () => {
+    if (!this.state.parsed) {
+      const { row } = this.props;
+      const parser = getParser(row.entry);
+      if (parser) {
+        // Use parser to highlight detected fields
+        const parsedFieldHighlights = [];
+        this.props.row.entry.replace(new RegExp(parser.fieldRegex, 'g'), substring => {
+          parsedFieldHighlights.push(substring.trim());
+          return '';
+        });
+        this.setState({ parsedFieldHighlights, parsed: true, parser });
+      }
+    }
+  };
+
+  render() {
+    const {
+      allRows,
+      highlighterExpressions,
+      onClickLabel,
+      row,
+      showDuplicates,
+      showLabels,
+      showLocalTime,
+      showUtc,
+    } = this.props;
+    const {
+      fieldCount,
+      fieldLabel,
+      fieldStats,
+      fieldValue,
+      parsed,
+      parsedFieldHighlights,
+      showFieldStats,
+    } = this.state;
+    const previewHighlights = highlighterExpressions && !_.isEqual(highlighterExpressions, row.searchWords);
+    const highlights = previewHighlights ? highlighterExpressions : row.searchWords;
+    const needsHighlighter = highlights && highlights.length > 0;
+    const highlightClassName = classnames('logs-row__match-highlight', {
+      'logs-row__match-highlight--preview': previewHighlights,
+    });
+    return (
+      <div className="logs-row">
+        {showDuplicates && (
+          <div className="logs-row__duplicates">{row.duplicates > 0 ? `${row.duplicates + 1}x` : null}</div>
         )}
+        <div className={row.logLevel ? `logs-row__level logs__row-level-${row.logLevel}` : ''} />
+        {showUtc && (
+          <div className="logs-row__time" title={`Local: ${row.timeLocal} (${row.timeFromNow})`}>
+            {row.timestamp}
+          </div>
+        )}
+        {showLocalTime && (
+          <div className="logs-row__time" title={`${row.timestamp} (${row.timeFromNow})`}>
+            {row.timeLocal}
+          </div>
+        )}
+        {showLabels && (
+          <div className="logs-row__labels">
+            <LogLabels allRows={allRows} labels={row.uniqueLabels} onClickLabel={onClickLabel} />
+          </div>
+        )}
+        <div className="logs-row__message" onMouseEnter={this.onMouseOverMessage} onMouseLeave={this.onMouseOutMessage}>
+          {parsed && (
+            <Highlighter
+              autoEscape
+              highlightTag={FieldHighlight(this.onClickHighlight)}
+              textToHighlight={row.entry}
+              searchWords={parsedFieldHighlights}
+              highlightClassName="logs-row__field-highlight"
+            />
+          )}
+          {!parsed &&
+            needsHighlighter && (
+              <Highlighter
+                textToHighlight={row.entry}
+                searchWords={highlights}
+                findChunks={findHighlightChunksInText}
+                highlightClassName={highlightClassName}
+              />
+            )}
+          {!parsed && !needsHighlighter && row.entry}
+          {showFieldStats && (
+            <Stats
+              stats={fieldStats}
+              label={fieldLabel}
+              value={fieldValue}
+              onClickClose={this.onClickClose}
+              rowCount={fieldCount}
+            />
+          )}
+        </div>
       </div>
-    </div>
+    );
-  );
+  }
 }
 
 function renderMetaItem(value: any, kind: LogsMetaKind) {

public/sass/components/_panel_logs.scss

@@ -158,6 +158,28 @@ $column-horizontal-spacing: 10px;
   text-align: right;
 }
 
+.logs-row__field-highlight {
+  // Undoing mark styling
+  background: inherit;
+  padding: inherit;
+  border-bottom: 1px dotted $typeahead-selected-color;
+
+  .logs-row__field-highlight--icon {
+    margin-left: 0.5em;
+    cursor: pointer;
+    display: none;
+  }
+}
+
+.logs-row__field-highlight:hover {
+  color: $typeahead-selected-color;
+  border-bottom-style: solid;
+
+  .logs-row__field-highlight--icon {
+    display: inline;
+  }
+}
+
 .logs-label {
   display: inline-block;
   padding: 0 2px;