Dashboard: Adds Logs Panel (alpha) as visualization option for Dashboards (#18641)

* WIP: intial commit

* Switch: Adds tooltip

* Refactor: Adds props to LogsPanelEditor

* Refactor: Moves LogRowContextProvider to grafana/ui

* Refactor: Moves LogRowContext and Alert to grafana/ui

* Refactor: Moves LogLabelStats to grafana/ui

* Refactor: Moves LogLabels and LogLabel to grafana/ui

* Refactor: Moves LogMessageAnsi and ansicolor to grafana/ui

* Refactor: Moves calculateFieldStats, LogsParsers and getParser to grafana/data

* Refactor: Moves findHighlightChunksInText to grafana/data

* Refactor: Moves LogRow to grafana/ui

* Refactor: Moving ExploreGraphPanel to grafana/ui

* Refactor: Copies Logs to grafana/ui

* Refactor: Moves ToggleButtonGroup to grafana/ui

* Refactor: Adds Logs to LogsPanel

* Refactor: Moves styles to emotion

* Feature: Adds LogsRows

* Refactor: Introduces render limit

* Styles: Moves styles to emotion

* Styles: Moves styles to emotion

* Styles: Moves styles to emotion

* Styles: Moves styles to emotion

* Refactor: Adds sorting to LogsPanelEditor

* Tests: Adds tests for sorting

* Refactor: Changes according to PR comments

* Refactor: Changes according to PR comments

* Refactor: Moves Logs and ExploreGraphPanel out of grafana/ui

* Fix: Shows the Show context label again

packages/grafana-data/src/types/logs.ts

@@ -105,3 +105,10 @@ export interface LogsParser {
    */
   test: (line: string) => any;
 }
+
+export enum LogsDedupDescription {
+  none = 'No de-duplication',
+  exact = 'De-duplication of successive lines that are identical, ignoring ISO datetimes.',
+  numbers = 'De-duplication of successive lines that are identical when ignoring numbers, e.g., IP addresses, latencies.',
+  signature = 'De-duplication of successive lines that have identical punctuation and whitespace.',
+}

packages/grafana-data/src/utils/index.ts

@@ -11,6 +11,7 @@ export * from './labels';
 export * from './object';
 export * from './moment_wrapper';
 export * from './thresholds';
+export * from './text';
 export * from './dataFrameHelper';
 export * from './dataFrameView';
 export * from './vector';

packages/grafana-data/src/utils/logs.test.ts

@@ -1,5 +1,5 @@
 import { LogLevel } from '../types/logs';
-import { getLogLevel } from './logs';
+import { getLogLevel, calculateLogsLabelStats, calculateFieldStats, getParser, LogsParsers } from './logs';
 
 describe('getLoglevel()', () => {
   it('returns no log level on empty line', () => {
@@ -25,3 +25,190 @@ describe('getLoglevel()', () => {
     expect(getLogLevel('WARN this could be a debug message')).toBe(LogLevel.warn);
   });
 });
+
+describe('calculateLogsLabelStats()', () => {
+  test('should return no stats for empty rows', () => {
+    expect(calculateLogsLabelStats([], '')).toEqual([]);
+  });
+
+  test('should return no stats of label is not found', () => {
+    const rows = [
+      {
+        entry: 'foo 1',
+        labels: {
+          foo: 'bar',
+        },
+      },
+    ];
+
+    expect(calculateLogsLabelStats(rows as any, 'baz')).toEqual([]);
+  });
+
+  test('should return stats for found labels', () => {
+    const rows = [
+      {
+        entry: 'foo 1',
+        labels: {
+          foo: 'bar',
+        },
+      },
+      {
+        entry: 'foo 0',
+        labels: {
+          foo: 'xxx',
+        },
+      },
+      {
+        entry: 'foo 2',
+        labels: {
+          foo: 'bar',
+        },
+      },
+    ];
+
+    expect(calculateLogsLabelStats(rows as any, 'foo')).toMatchObject([
+      {
+        value: 'bar',
+        count: 2,
+      },
+      {
+        value: 'xxx',
+        count: 1,
+      },
+    ]);
+  });
+});
+
+describe('LogsParsers', () => {
+  describe('logfmt', () => {
+    const parser = LogsParsers.logfmt;
+
+    test('should detect format', () => {
+      expect(parser.test('foo')).toBeFalsy();
+      expect(parser.test('foo=bar')).toBeTruthy();
+    });
+
+    test('should return parsed fields', () => {
+      expect(parser.getFields('foo=bar baz="42 + 1"')).toEqual(['foo=bar', 'baz="42 + 1"']);
+    });
+
+    test('should return label for field', () => {
+      expect(parser.getLabelFromField('foo=bar')).toBe('foo');
+    });
+
+    test('should return value for field', () => {
+      expect(parser.getValueFromField('foo=bar')).toBe('bar');
+    });
+
+    test('should build a valid value matcher', () => {
+      const matcher = parser.buildMatcher('foo');
+      const match = 'foo=bar'.match(matcher);
+      expect(match).toBeDefined();
+      expect(match![1]).toBe('bar');
+    });
+  });
+
+  describe('JSON', () => {
+    const parser = LogsParsers.JSON;
+
+    test('should detect format', () => {
+      expect(parser.test('foo')).toBeFalsy();
+      expect(parser.test('{"foo":"bar"}')).toBeTruthy();
+    });
+
+    test('should return parsed fields', () => {
+      expect(parser.getFields('{ "foo" : "bar", "baz" : 42 }')).toEqual(['"foo" : "bar"', '"baz" : 42']);
+    });
+
+    test('should return parsed fields for nested quotes', () => {
+      expect(parser.getFields(`{"foo":"bar: '[value=\\"42\\"]'"}`)).toEqual([`"foo":"bar: '[value=\\"42\\"]'"`]);
+    });
+
+    test('should return label for field', () => {
+      expect(parser.getLabelFromField('"foo" : "bar"')).toBe('foo');
+    });
+
+    test('should return value for field', () => {
+      expect(parser.getValueFromField('"foo" : "bar"')).toBe('"bar"');
+      expect(parser.getValueFromField('"foo" : 42')).toBe('42');
+      expect(parser.getValueFromField('"foo" : 42.1')).toBe('42.1');
+    });
+
+    test('should build a valid value matcher for strings', () => {
+      const matcher = parser.buildMatcher('foo');
+      const match = '{"foo":"bar"}'.match(matcher);
+      expect(match).toBeDefined();
+      expect(match![1]).toBe('bar');
+    });
+
+    test('should build a valid value matcher for integers', () => {
+      const matcher = parser.buildMatcher('foo');
+      const match = '{"foo":42.1}'.match(matcher);
+      expect(match).toBeDefined();
+      expect(match![1]).toBe('42.1');
+    });
+  });
+});
+
+describe('calculateFieldStats()', () => {
+  test('should return no stats for empty rows', () => {
+    expect(calculateFieldStats([], /foo=(.*)/)).toEqual([]);
+  });
+
+  test('should return no stats if extractor does not match', () => {
+    const rows = [
+      {
+        entry: 'foo=bar',
+      },
+    ];
+
+    expect(calculateFieldStats(rows as any, /baz=(.*)/)).toEqual([]);
+  });
+
+  test('should return stats for found field', () => {
+    const rows = [
+      {
+        entry: 'foo="42 + 1"',
+      },
+      {
+        entry: 'foo=503 baz=foo',
+      },
+      {
+        entry: 'foo="42 + 1"',
+      },
+      {
+        entry: 't=2018-12-05T07:44:59+0000 foo=503',
+      },
+    ];
+
+    expect(calculateFieldStats(rows as any, /foo=("[^"]*"|\S+)/)).toMatchObject([
+      {
+        value: '"42 + 1"',
+        count: 2,
+      },
+      {
+        value: '503',
+        count: 2,
+      },
+    ]);
+  });
+});
+
+describe('getParser()', () => {
+  test('should return no parser on empty line', () => {
+    expect(getParser('')).toBeUndefined();
+  });
+
+  test('should return no parser on unknown line pattern', () => {
+    expect(getParser('To Be or not to be')).toBeUndefined();
+  });
+
+  test('should return logfmt parser on key value patterns', () => {
+    expect(getParser('foo=bar baz="41 + 1')).toEqual(LogsParsers.logfmt);
+  });
+
+  test('should return JSON parser on JSON log lines', () => {
+    // TODO implement other JSON value types than string
+    expect(getParser('{"foo": "bar", "baz": "41 + 1"}')).toEqual(LogsParsers.JSON);
+  });
+});

packages/grafana-data/src/utils/logs.ts

@@ -1,7 +1,11 @@
-import { LogLevel } from '../types/logs';
+import { countBy, chain, map, escapeRegExp } from 'lodash';
+
+import { LogLevel, LogRowModel, LogLabelStatsModel, LogsParser } from '../types/logs';
 import { DataFrame, FieldType } from '../types/index';
 import { ArrayVector } from './vector';
 
+const LOGFMT_REGEXP = /(?:^|\s)(\w+)=("[^"]*"|\S+)/;
+
 /**
  * Returns the log level of a log line.
  * Parse the line for level words. If no level is found, it returns `LogLevel.unknown`.
@@ -54,3 +58,98 @@ export function addLogLevelToSeries(series: DataFrame, lineIndex: number): DataF
     ],
   };
 }
+
+export function calculateLogsLabelStats(rows: LogRowModel[], label: string): LogLabelStatsModel[] {
+  // Consider only rows that have the given label
+  const rowsWithLabel = rows.filter(row => row.labels[label] !== undefined);
+  const rowCount = rowsWithLabel.length;
+
+  // Get label value counts for eligible rows
+  const countsByValue = countBy(rowsWithLabel, row => (row as LogRowModel).labels[label]);
+  const sortedCounts = chain(countsByValue)
+    .map((count, value) => ({ count, value, proportion: count / rowCount }))
+    .sortBy('count')
+    .reverse()
+    .value();
+
+  return sortedCounts;
+}
+
+export const LogsParsers: { [name: string]: LogsParser } = {
+  JSON: {
+    buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"?([\\d\\.]+|[^"]*)"?`),
+    getFields: line => {
+      const fields: string[] = [];
+      try {
+        const parsed = JSON.parse(line);
+        map(parsed, (value, key) => {
+          const fieldMatcher = new RegExp(`"${key}"\\s*:\\s*"?${escapeRegExp(JSON.stringify(value))}"?`);
+
+          const match = line.match(fieldMatcher);
+          if (match) {
+            fields.push(match[0]);
+          }
+        });
+      } catch {}
+      return fields;
+    },
+    getLabelFromField: field => (field.match(/^"(\w+)"\s*:/) || [])[1],
+    getValueFromField: field => (field.match(/:\s*(.*)$/) || [])[1],
+    test: line => {
+      try {
+        return JSON.parse(line);
+      } catch (error) {}
+    },
+  },
+
+  logfmt: {
+    buildMatcher: label => new RegExp(`(?:^|\\s)${label}=("[^"]*"|\\S+)`),
+    getFields: line => {
+      const fields: string[] = [];
+      line.replace(new RegExp(LOGFMT_REGEXP, 'g'), substring => {
+        fields.push(substring.trim());
+        return '';
+      });
+      return fields;
+    },
+    getLabelFromField: field => (field.match(LOGFMT_REGEXP) || [])[1],
+    getValueFromField: field => (field.match(LOGFMT_REGEXP) || [])[2],
+    test: line => LOGFMT_REGEXP.test(line),
+  },
+};
+
+export function calculateFieldStats(rows: LogRowModel[], extractor: RegExp): LogLabelStatsModel[] {
+  // Consider only rows that satisfy the matcher
+  const rowsWithField = rows.filter(row => extractor.test(row.entry));
+  const rowCount = rowsWithField.length;
+
+  // Get field value counts for eligible rows
+  const countsByValue = countBy(rowsWithField, r => {
+    const row: LogRowModel = r;
+    const match = row.entry.match(extractor);
+
+    return match ? match[1] : null;
+  });
+  const sortedCounts = chain(countsByValue)
+    .map((count, value) => ({ count, value, proportion: count / rowCount }))
+    .sortBy('count')
+    .reverse()
+    .value();
+
+  return sortedCounts;
+}
+
+export function getParser(line: string): LogsParser | undefined {
+  let parser;
+  try {
+    if (LogsParsers.JSON.test(line)) {
+      parser = LogsParsers.JSON;
+    }
+  } catch (error) {}
+
+  if (!parser && LogsParsers.logfmt.test(line)) {
+    parser = LogsParsers.logfmt;
+  }
+
+  return parser;
+}

packages/grafana-data/src/utils/text.test.ts

@@ -0,0 +1,68 @@
+import { findMatchesInText, parseFlags } from './text';
+
+describe('findMatchesInText()', () => {
+  it('gets no matches for when search and or line are empty', () => {
+    expect(findMatchesInText('', '')).toEqual([]);
+    expect(findMatchesInText('foo', '')).toEqual([]);
+    expect(findMatchesInText('', 'foo')).toEqual([]);
+  });
+
+  it('gets no matches for unmatched search string', () => {
+    expect(findMatchesInText('foo', 'bar')).toEqual([]);
+  });
+
+  it('gets matches for matched search string', () => {
+    expect(findMatchesInText('foo', 'foo')).toEqual([{ length: 3, start: 0, text: 'foo', end: 3 }]);
+    expect(findMatchesInText(' foo ', 'foo')).toEqual([{ length: 3, start: 1, text: 'foo', end: 4 }]);
+  });
+
+  test('should find all matches for a complete regex', () => {
+    expect(findMatchesInText(' foo foo bar ', 'foo|bar')).toEqual([
+      { length: 3, start: 1, text: 'foo', end: 4 },
+      { length: 3, start: 5, text: 'foo', end: 8 },
+      { length: 3, start: 9, text: 'bar', end: 12 },
+    ]);
+  });
+
+  test('not fail on incomplete regex', () => {
+    expect(findMatchesInText(' foo foo bar ', 'foo|')).toEqual([
+      { length: 3, start: 1, text: 'foo', end: 4 },
+      { length: 3, start: 5, text: 'foo', end: 8 },
+    ]);
+    expect(findMatchesInText('foo foo bar', '(')).toEqual([]);
+    expect(findMatchesInText('foo foo bar', '(foo|')).toEqual([]);
+  });
+
+  test('should parse and use flags', () => {
+    expect(findMatchesInText(' foo FOO bar ', '(?i)foo')).toEqual([
+      { length: 3, start: 1, text: 'foo', end: 4 },
+      { length: 3, start: 5, text: 'FOO', end: 8 },
+    ]);
+    expect(findMatchesInText(' foo FOO bar ', '(?i)(?-i)foo')).toEqual([{ length: 3, start: 1, text: 'foo', end: 4 }]);
+    expect(findMatchesInText('FOO\nfoobar\nbar', '(?ims)^foo.')).toEqual([
+      { length: 4, start: 0, text: 'FOO\n', end: 4 },
+      { length: 4, start: 4, text: 'foob', end: 8 },
+    ]);
+    expect(findMatchesInText('FOO\nfoobar\nbar', '(?ims)(?-smi)^foo.')).toEqual([]);
+  });
+});
+
+describe('parseFlags()', () => {
+  it('when no flags or text', () => {
+    expect(parseFlags('')).toEqual({ cleaned: '', flags: 'g' });
+    expect(parseFlags('(?is)')).toEqual({ cleaned: '', flags: 'gis' });
+    expect(parseFlags('foo')).toEqual({ cleaned: 'foo', flags: 'g' });
+  });
+
+  it('when flags present', () => {
+    expect(parseFlags('(?i)foo')).toEqual({ cleaned: 'foo', flags: 'gi' });
+    expect(parseFlags('(?ims)foo')).toEqual({ cleaned: 'foo', flags: 'gims' });
+  });
+
+  it('when flags cancel each other', () => {
+    expect(parseFlags('(?i)(?-i)foo')).toEqual({ cleaned: 'foo', flags: 'g' });
+    expect(parseFlags('(?i-i)foo')).toEqual({ cleaned: 'foo', flags: 'g' });
+    expect(parseFlags('(?is)(?-ims)foo')).toEqual({ cleaned: 'foo', flags: 'g' });
+    expect(parseFlags('(?i)(?-i)(?i)foo')).toEqual({ cleaned: 'foo', flags: 'gi' });
+  });
+});

packages/grafana-data/src/utils/text.ts

@@ -0,0 +1,84 @@
+export interface TextMatch {
+  text: string;
+  start: number;
+  length: number;
+  end: number;
+}
+
+/**
+ * Adapt findMatchesInText for react-highlight-words findChunks handler.
+ * See https://github.com/bvaughn/react-highlight-words#props
+ */
+export function findHighlightChunksInText({
+  searchWords,
+  textToHighlight,
+}: {
+  searchWords: string[];
+  textToHighlight: string;
+}) {
+  return searchWords.reduce((acc: any, term: string) => [...acc, ...findMatchesInText(textToHighlight, term)], []);
+}
+
+const cleanNeedle = (needle: string): string => {
+  return needle.replace(/[[{(][\w,.-?:*+]+$/, '');
+};
+
+/**
+ * Returns a list of substring regexp matches.
+ */
+export function findMatchesInText(haystack: string, needle: string): TextMatch[] {
+  // Empty search can send re.exec() into infinite loop, exit early
+  if (!haystack || !needle) {
+    return [];
+  }
+  const matches: TextMatch[] = [];
+  const { cleaned, flags } = parseFlags(cleanNeedle(needle));
+  let regexp: RegExp;
+  try {
+    regexp = new RegExp(`(?:${cleaned})`, flags);
+  } catch (error) {
+    return matches;
+  }
+  haystack.replace(regexp, (substring, ...rest) => {
+    if (substring) {
+      const offset = rest[rest.length - 2];
+      matches.push({
+        text: substring,
+        start: offset,
+        length: substring.length,
+        end: offset + substring.length,
+      });
+    }
+    return '';
+  });
+  return matches;
+}
+
+const CLEAR_FLAG = '-';
+const FLAGS_REGEXP = /\(\?([ims-]+)\)/g;
+
+/**
+ * Converts any mode modifers in the text to the Javascript equivalent flag
+ */
+export function parseFlags(text: string): { cleaned: string; flags: string } {
+  const flags: Set<string> = new Set(['g']);
+
+  const cleaned = text.replace(FLAGS_REGEXP, (str, group) => {
+    const clearAll = group.startsWith(CLEAR_FLAG);
+
+    for (let i = 0; i < group.length; ++i) {
+      const flag = group.charAt(i);
+      if (clearAll || group.charAt(i - 1) === CLEAR_FLAG) {
+        flags.delete(flag);
+      } else if (flag !== CLEAR_FLAG) {
+        flags.add(flag);
+      }
+    }
+    return ''; // Remove flag from text
+  });
+
+  return {
+    cleaned: cleaned,
+    flags: Array.from(flags).join(''),
+  };
+}