Merge pull request #14438 from grafana/davkal/explore-enhanced-log-parsing

Explore: Improved line parsing for logging

public/app/core/logs_model.ts

@@ -108,11 +108,21 @@ export interface LogsParser {
    * Used to filter rows, and first capture group contains the value.
    */
   buildMatcher: (label: string) => RegExp;
+
   /**
-   * Regex to find a field in the log line.
-   * First capture group contains the label value, second capture group the value.
+   * Returns all parsable substrings from a line, used for highlighting
    */
-  fieldRegex: RegExp;
+  getFields: (line: string) => string[];
+
+  /**
+   * Gets the label name from a parsable substring of a line
+   */
+  getLabelFromField: (field: string) => string;
+
+  /**
+   * Gets the label value from a parsable substring of a line
+   */
+  getValueFromField: (field: string) => string;
   /**
    * Function to verify if this is a valid parser for the given line.
    * The parser accepts the line unless it returns undefined.
@@ -120,20 +130,48 @@ export interface LogsParser {
   test: (line: string) => any;
 }
 
+const LOGFMT_REGEXP = /(?:^|\s)(\w+)=("[^"]*"|\S+)/;
+
 export const LogsParsers: { [name: string]: LogsParser } = {
   JSON: {
-    buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"([^"]*)"`),
-    fieldRegex: /"(\w+)"\s*:\s*"([^"]*)"/,
+    buildMatcher: label => new RegExp(`(?:{|,)\\s*"${label}"\\s*:\\s*"?([\\d\\.]+|[^"]*)"?`),
+    getFields: line => {
+      const fields = [];
+      try {
+        const parsed = JSON.parse(line);
+        _.map(parsed, (value, key) => {
+          const fieldMatcher = new RegExp(`"${key}"\\s*:\\s*"?${_.escapeRegExp(JSON.stringify(value))}"?`);
+
+          const match = line.match(fieldMatcher);
+          if (match) {
+            fields.push(match[0]);
+          }
+        });
+      } catch {}
+      return fields;
+    },
+    getLabelFromField: field => (field.match(/^"(\w+)"\s*:/) || [])[1],
+    getValueFromField: field => (field.match(/:\s*(.*)$/) || [])[1],
     test: line => {
       try {
         return JSON.parse(line);
       } catch (error) {}
     },
   },
+
   logfmt: {
     buildMatcher: label => new RegExp(`(?:^|\\s)${label}=("[^"]*"|\\S+)`),
-    fieldRegex: /(?:^|\s)(\w+)=("[^"]*"|\S+)/,
-    test: line => LogsParsers.logfmt.fieldRegex.test(line),
+    getFields: line => {
+      const fields = [];
+      line.replace(new RegExp(LOGFMT_REGEXP, 'g'), substring => {
+        fields.push(substring.trim());
+        return '';
+      });
+      return fields;
+    },
+    getLabelFromField: field => (field.match(LOGFMT_REGEXP) || [])[1],
+    getValueFromField: field => (field.match(LOGFMT_REGEXP) || [])[2],
+    test: line => LOGFMT_REGEXP.test(line),
   },
 };
 

public/app/core/specs/logs_model.test.ts

@@ -240,11 +240,16 @@ describe('LogsParsers', () => {
       expect(parser.test('foo=bar')).toBeTruthy();
     });
 
-    test('should have a valid fieldRegex', () => {
-      const match = 'foo=bar'.match(parser.fieldRegex);
-      expect(match).toBeDefined();
-      expect(match[1]).toBe('foo');
-      expect(match[2]).toBe('bar');
+    test('should return parsed fields', () => {
+      expect(parser.getFields('foo=bar baz="42 + 1"')).toEqual(['foo=bar', 'baz="42 + 1"']);
+    });
+
+    test('should return label for field', () => {
+      expect(parser.getLabelFromField('foo=bar')).toBe('foo');
+    });
+
+    test('should return value for field', () => {
+      expect(parser.getValueFromField('foo=bar')).toBe('bar');
     });
 
     test('should build a valid value matcher', () => {
@@ -263,18 +268,36 @@ describe('LogsParsers', () => {
       expect(parser.test('{"foo":"bar"}')).toBeTruthy();
     });
 
-    test('should have a valid fieldRegex', () => {
-      const match = '{"foo":"bar"}'.match(parser.fieldRegex);
-      expect(match).toBeDefined();
-      expect(match[1]).toBe('foo');
-      expect(match[2]).toBe('bar');
+    test('should return parsed fields', () => {
+      expect(parser.getFields('{ "foo" : "bar", "baz" : 42 }')).toEqual(['"foo" : "bar"', '"baz" : 42']);
     });
 
-    test('should build a valid value matcher', () => {
+    test('should return parsed fields for nested quotes', () => {
+      expect(parser.getFields(`{"foo":"bar: '[value=\\"42\\"]'"}`)).toEqual([`"foo":"bar: '[value=\\"42\\"]'"`]);
+    });
+
+    test('should return label for field', () => {
+      expect(parser.getLabelFromField('"foo" : "bar"')).toBe('foo');
+    });
+
+    test('should return value for field', () => {
+      expect(parser.getValueFromField('"foo" : "bar"')).toBe('"bar"');
+      expect(parser.getValueFromField('"foo" : 42')).toBe('42');
+      expect(parser.getValueFromField('"foo" : 42.1')).toBe('42.1');
+    });
+
+    test('should build a valid value matcher for strings', () => {
       const matcher = parser.buildMatcher('foo');
       const match = '{"foo":"bar"}'.match(matcher);
       expect(match).toBeDefined();
       expect(match[1]).toBe('bar');
     });
+
+    test('should build a valid value matcher for integers', () => {
+      const matcher = parser.buildMatcher('foo');
+      const match = '{"foo":42.1}'.match(matcher);
+      expect(match).toBeDefined();
+      expect(match[1]).toBe('42.1');
+    });
   });
 });

public/app/features/explore/Logs.tsx

@@ -73,7 +73,7 @@ interface RowState {
   fieldStats: LogsLabelStat[];
   fieldValue: string;
   parsed: boolean;
-  parser: LogsParser;
+  parser?: LogsParser;
   parsedFieldHighlights: string[];
   showFieldStats: boolean;
 }
@@ -94,7 +94,7 @@ class Row extends PureComponent<RowProps, RowState> {
     fieldStats: null,
     fieldValue: null,
     parsed: false,
-    parser: null,
+    parser: undefined,
     parsedFieldHighlights: [],
     showFieldStats: false,
   };
@@ -110,19 +110,16 @@ class Row extends PureComponent<RowProps, RowState> {
   onClickHighlight = (fieldText: string) => {
     const { getRows } = this.props;
     const { parser } = this.state;
+    const allRows = getRows();
 
-    const fieldMatch = fieldText.match(parser.fieldRegex);
-    if (fieldMatch) {
-      const allRows = getRows();
-      // Build value-agnostic row matcher based on the field label
-      const fieldLabel = fieldMatch[1];
-      const fieldValue = fieldMatch[2];
-      const matcher = parser.buildMatcher(fieldLabel);
-      const fieldStats = calculateFieldStats(allRows, matcher);
-      const fieldCount = fieldStats.reduce((sum, stat) => sum + stat.count, 0);
+    // Build value-agnostic row matcher based on the field label
+    const fieldLabel = parser.getLabelFromField(fieldText);
+    const fieldValue = parser.getValueFromField(fieldText);
+    const matcher = parser.buildMatcher(fieldLabel);
+    const fieldStats = calculateFieldStats(allRows, matcher);
+    const fieldCount = fieldStats.reduce((sum, stat) => sum + stat.count, 0);
 
-      this.setState({ fieldCount, fieldLabel, fieldStats, fieldValue, showFieldStats: true });
-    }
+    this.setState({ fieldCount, fieldLabel, fieldStats, fieldValue, showFieldStats: true });
   };
 
   onMouseOverMessage = () => {
@@ -141,11 +138,7 @@ class Row extends PureComponent<RowProps, RowState> {
       const parser = getParser(row.entry);
       if (parser) {
         // Use parser to highlight detected fields
-        const parsedFieldHighlights = [];
-        this.props.row.entry.replace(new RegExp(parser.fieldRegex, 'g'), substring => {
-          parsedFieldHighlights.push(substring.trim());
-          return '';
-        });
+        const parsedFieldHighlights = parser.getFields(this.props.row.entry);
         this.setState({ parsedFieldHighlights, parsed: true, parser });
       }
     }