Merge pull request #14118 from grafana/davkal/explore-logs-dedup

Explore: POC dedup logging rows

public/app/core/logs_model.ts

@@ -31,6 +31,7 @@ export interface LogSearchMatch {
 }
 
 export interface LogRow {
+  duplicates?: number;
   entry: string;
   key: string; // timestamp + labels
   labels: string;
@@ -71,6 +72,53 @@ export interface LogsStreamLabels {
   [key: string]: string;
 }
 
+export enum LogsDedupStrategy {
+  none = 'none',
+  exact = 'exact',
+  numbers = 'numbers',
+  signature = 'signature',
+}
+
+const isoDateRegexp = /\d{4}-[01]\d-[0-3]\dT[0-2]\d:[0-5]\d:[0-6]\d[,\.]\d+([+-][0-2]\d:[0-5]\d|Z)/g;
+function isDuplicateRow(row: LogRow, other: LogRow, strategy: LogsDedupStrategy): boolean {
+  switch (strategy) {
+    case LogsDedupStrategy.exact:
+      // Exact still strips dates
+      return row.entry.replace(isoDateRegexp, '') === other.entry.replace(isoDateRegexp, '');
+
+    case LogsDedupStrategy.numbers:
+      return row.entry.replace(/\d/g, '') === other.entry.replace(/\d/g, '');
+
+    case LogsDedupStrategy.signature:
+      return row.entry.replace(/\w/g, '') === other.entry.replace(/\w/g, '');
+
+    default:
+      return false;
+  }
+}
+
+export function dedupLogRows(logs: LogsModel, strategy: LogsDedupStrategy): LogsModel {
+  if (strategy === LogsDedupStrategy.none) {
+    return logs;
+  }
+
+  const dedupedRows = logs.rows.reduce((result: LogRow[], row: LogRow, index, list) => {
+    const previous = result[result.length - 1];
+    if (index > 0 && isDuplicateRow(row, previous, strategy)) {
+      previous.duplicates++;
+    } else {
+      row.duplicates = 0;
+      result.push(row);
+    }
+    return result;
+  }, []);
+
+  return {
+    ...logs,
+    rows: dedupedRows,
+  };
+}
+
 export function makeSeriesForLogs(rows: LogRow[], intervalMs: number): TimeSeries[] {
   // Graph time series by log level
   const seriesByLevel = {};

public/app/core/specs/logs_model.test.ts

@@ -0,0 +1,108 @@
+import { dedupLogRows, LogsDedupStrategy, LogsModel } from '../logs_model';
+
+describe('dedupLogRows()', () => {
+  test('should return rows as is when dedup is set to none', () => {
+    const logs = {
+      rows: [
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+      ],
+    };
+    expect(dedupLogRows(logs as LogsModel, LogsDedupStrategy.none).rows).toMatchObject(logs.rows);
+  });
+
+  test('should dedup on exact matches', () => {
+    const logs = {
+      rows: [
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+        {
+          entry: 'INFO test 2.44 on [xxx]',
+        },
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+      ],
+    };
+    expect(dedupLogRows(logs as LogsModel, LogsDedupStrategy.exact).rows).toEqual([
+      {
+        duplicates: 1,
+        entry: 'WARN test 1.23 on [xxx]',
+      },
+      {
+        duplicates: 0,
+        entry: 'INFO test 2.44 on [xxx]',
+      },
+      {
+        duplicates: 0,
+        entry: 'WARN test 1.23 on [xxx]',
+      },
+    ]);
+  });
+
+  test('should dedup on number matches', () => {
+    const logs = {
+      rows: [
+        {
+          entry: 'WARN test 1.2323423 on [xxx]',
+        },
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+        {
+          entry: 'INFO test 2.44 on [xxx]',
+        },
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+      ],
+    };
+    expect(dedupLogRows(logs as LogsModel, LogsDedupStrategy.numbers).rows).toEqual([
+      {
+        duplicates: 1,
+        entry: 'WARN test 1.2323423 on [xxx]',
+      },
+      {
+        duplicates: 0,
+        entry: 'INFO test 2.44 on [xxx]',
+      },
+      {
+        duplicates: 0,
+        entry: 'WARN test 1.23 on [xxx]',
+      },
+    ]);
+  });
+
+  test('should dedup on signature matches', () => {
+    const logs = {
+      rows: [
+        {
+          entry: 'WARN test 1.2323423 on [xxx]',
+        },
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+        {
+          entry: 'INFO test 2.44 on [xxx]',
+        },
+        {
+          entry: 'WARN test 1.23 on [xxx]',
+        },
+      ],
+    };
+    expect(dedupLogRows(logs as LogsModel, LogsDedupStrategy.signature).rows).toEqual([
+      {
+        duplicates: 3,
+        entry: 'WARN test 1.2323423 on [xxx]',
+      },
+    ]);
+  });
+});

public/app/features/explore/Logs.tsx

@@ -2,7 +2,7 @@ import React, { Fragment, PureComponent } from 'react';
 import Highlighter from 'react-highlight-words';
 
 import { RawTimeRange } from 'app/types/series';
-import { LogsModel } from 'app/core/logs_model';
+import { LogsDedupStrategy, LogsModel, dedupLogRows } from 'app/core/logs_model';
 import { findHighlightChunksInText } from 'app/core/utils/text';
 import { Switch } from 'app/core/components/Switch/Switch';
 
@@ -32,6 +32,7 @@ interface LogsProps {
 }
 
 interface LogsState {
+  dedup: LogsDedupStrategy;
   showLabels: boolean;
   showLocalTime: boolean;
   showUtc: boolean;
@@ -39,11 +40,21 @@ interface LogsState {
 
 export default class Logs extends PureComponent<LogsProps, LogsState> {
   state = {
+    dedup: LogsDedupStrategy.none,
     showLabels: true,
     showLocalTime: true,
     showUtc: false,
   };
 
+  onChangeDedup = (dedup: LogsDedupStrategy) => {
+    this.setState(prevState => {
+      if (prevState.dedup === dedup) {
+        return { dedup: LogsDedupStrategy.none };
+      }
+      return { dedup };
+    });
+  };
+
   onChangeLabels = (event: React.SyntheticEvent) => {
     const target = event.target as HTMLInputElement;
     this.setState({
@@ -67,9 +78,18 @@ export default class Logs extends PureComponent<LogsProps, LogsState> {
 
   render() {
     const { className = '', data, loading = false, position, range } = this.props;
-    const { showLabels, showLocalTime, showUtc } = this.state;
+    const { dedup, showLabels, showLocalTime, showUtc } = this.state;
     const hasData = data && data.rows && data.rows.length > 0;
-    const cssColumnSizes = ['4px'];
+    const dedupedData = dedupLogRows(data, dedup);
+    const dedupCount = dedupedData.rows.reduce((sum, row) => sum + row.duplicates, 0);
+    const meta = [...data.meta];
+    if (dedup !== LogsDedupStrategy.none) {
+      meta.push({
+        label: 'Dedup count',
+        value: String(dedupCount),
+      });
+    }
+    const cssColumnSizes = ['3px']; // Log-level indicator line
     if (showUtc) {
       cssColumnSizes.push('minmax(100px, max-content)');
     }
@@ -102,10 +122,34 @@ export default class Logs extends PureComponent<LogsProps, LogsState> {
             <Switch label="Timestamp" checked={showUtc} onChange={this.onChangeUtc} small />
             <Switch label="Local time" checked={showLocalTime} onChange={this.onChangeLocalTime} small />
             <Switch label="Labels" checked={showLabels} onChange={this.onChangeLabels} small />
+            <Switch
+              label="Dedup: off"
+              checked={dedup === LogsDedupStrategy.none}
+              onChange={() => this.onChangeDedup(LogsDedupStrategy.none)}
+              small
+            />
+            <Switch
+              label="Dedup: exact"
+              checked={dedup === LogsDedupStrategy.exact}
+              onChange={() => this.onChangeDedup(LogsDedupStrategy.exact)}
+              small
+            />
+            <Switch
+              label="Dedup: numbers"
+              checked={dedup === LogsDedupStrategy.numbers}
+              onChange={() => this.onChangeDedup(LogsDedupStrategy.numbers)}
+              small
+            />
+            <Switch
+              label="Dedup: signature"
+              checked={dedup === LogsDedupStrategy.signature}
+              onChange={() => this.onChangeDedup(LogsDedupStrategy.signature)}
+              small
+            />
             {hasData &&
-              data.meta && (
+              meta && (
                 <div className="logs-meta">
-                  {data.meta.map(item => (
+                  {meta.map(item => (
                     <div className="logs-meta-item" key={item.label}>
                       <span className="logs-meta-item__label">{item.label}:</span>
                       <span className="logs-meta-item__value">{item.value}</span>
@@ -118,9 +162,17 @@ export default class Logs extends PureComponent<LogsProps, LogsState> {
 
         <div className="logs-entries" style={logEntriesStyle}>
           {hasData &&
-            data.rows.map(row => (
+            dedupedData.rows.map(row => (
               <Fragment key={row.key}>
-                <div className={row.logLevel ? `logs-row-level logs-row-level-${row.logLevel}` : ''} />
+                <div className={row.logLevel ? `logs-row-level logs-row-level-${row.logLevel}` : ''}>
+                  {row.duplicates > 0 && (
+                    <div className="logs-row-level__duplicates" title={`${row.duplicates} duplicates`}>
+                      {Array.apply(null, { length: row.duplicates }).map(index => (
+                        <div className="logs-row-level__duplicate" key={`${index}`} />
+                      ))}
+                    </div>
+                  )}
+                </div>
                 {showUtc && <div title={`Local: ${row.timeLocal} (${row.timeFromNow})`}>{row.timestamp}</div>}
                 {showLocalTime && <div title={`${row.timestamp} (${row.timeFromNow})`}>{row.timeLocal}</div>}
                 {showLabels && (

public/sass/pages/_explore.scss

@@ -300,8 +300,8 @@
 
     .logs-row-level {
       background-color: transparent;
-      margin: 6px 0;
+      margin: 2px 0;
-      border-radius: 2px;
+      position: relative;
       opacity: 0.8;
     }
 
@@ -326,6 +326,25 @@
     .logs-row-level-debug {
       background-color: #1f78c1;
     }
+
+    .logs-row-level__duplicates {
+      position: absolute;
+      width: 9px;
+      height: 100%;
+      top: 0;
+      left: 5px;
+      display: flex;
+      flex-wrap: wrap;
+      align-items: flex-start;
+      align-content: flex-start;
+    }
+
+    .logs-row-level__duplicate {
+      width: 2px;
+      height: 3px;
+      background-color: #1f78c1;
+      margin: 0 1px 1px 0;
+    }
   }
 }