Explore: Logging graph overview and view options

- Logging gets a graph for log distribution (currently per stream, but I
  think I'll change that to per log-level)
- added grid columns for timestamp and unique labels
- show common labels of streams
- View options to show/hide time columns, label columns
- created `--small` modifier for Switch CSS classes
- merging of streams is now a datasource responsibility

public/app/core/components/Switch/Switch.tsx

@@ -5,6 +5,7 @@ export interface Props {
   label: string;
   checked: boolean;
   labelClass?: string;
+  small?: boolean;
   switchClass?: string;
   onChange: (event) => any;
 }
@@ -24,10 +25,14 @@ export class Switch extends PureComponent<Props, State> {
   };
 
   render() {
-    const { labelClass, switchClass, label, checked } = this.props;
+    const { labelClass = '', switchClass = '', label, checked, small } = this.props;
     const labelId = `check-${this.state.id}`;
-    const labelClassName = `gf-form-label ${labelClass} pointer`;
+    let labelClassName = `gf-form-label ${labelClass} pointer`;
-    const switchClassName = `gf-form-switch ${switchClass}`;
+    let switchClassName = `gf-form-switch ${switchClass}`;
+    if (small) {
+      labelClassName += ' gf-form-label--small';
+      switchClassName += ' gf-form-switch--small';
+    }
 
     return (
       <div className="gf-form">

public/app/core/logs_model.ts

@@ -1,4 +1,5 @@
 import _ from 'lodash';
+import { TimeSeries } from 'app/core/core';
 
 export enum LogLevel {
   crit = 'crit',
@@ -19,25 +20,34 @@ export interface LogSearchMatch {
 export interface LogRow {
   key: string;
   entry: string;
+  labels: string;
   logLevel: LogLevel;
   timestamp: string;
   timeFromNow: string;
+  timeJs: number;
   timeLocal: string;
   searchWords?: string[];
 }
 
-export interface LogsModel {
+export interface LogsMetaItem {
-  rows: LogRow[];
+  label: string;
+  value: string;
 }
 
-export function mergeStreams(streams: LogsModel[], limit?: number): LogsModel {
+export interface LogsModel {
-  const combinedEntries = streams.reduce((acc, stream) => {
+  meta?: LogsMetaItem[];
-    return [...acc, ...stream.rows];
+  rows: LogRow[];
-  }, []);
+  series?: TimeSeries[];
-  const sortedEntries = _.chain(combinedEntries)
+}
-    .sortBy('timestamp')
+
-    .reverse()
+export interface LogsStream {
-    .slice(0, limit || combinedEntries.length)
+  labels: string;
-    .value();
+  entries: LogsStreamEntry[];
-  return { rows: sortedEntries };
+  parsedLabels: { [key: string]: string };
+  graphSeries: TimeSeries;
+}
+
+export interface LogsStreamEntry {
+  line: string;
+  timestamp: string;
 }

public/app/features/explore/Explore.tsx

@@ -25,7 +25,6 @@ import ErrorBoundary from './ErrorBoundary';
 import TimePicker from './TimePicker';
 import { ensureQueries, generateQueryKey, hasQuery } from './utils/query';
 import { DataSource } from 'app/types/datasources';
-import { mergeStreams } from 'app/core/logs_model';
 
 const MAX_HISTORY_ITEMS = 100;
 
@@ -770,9 +769,14 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
       new TableModel(),
       ...queryTransactions.filter(qt => qt.resultType === 'Table' && qt.done && qt.result).map(qt => qt.result)
     );
-    const logsResult = mergeStreams(
+    const logsResult =
-      queryTransactions.filter(qt => qt.resultType === 'Logs' && qt.done && qt.result).map(qt => qt.result)
+      datasource && datasource.mergeStreams
-    );
+        ? datasource.mergeStreams(
+            _.flatten(
+              queryTransactions.filter(qt => qt.resultType === 'Logs' && qt.done && qt.result).map(qt => qt.result)
+            )
+          )
+        : undefined;
     const loading = queryTransactions.some(qt => !qt.done);
     const showStartPages = StartPage && queryTransactions.length === 0;
     const viewModeCount = [supportsGraph, supportsLogs, supportsTable].filter(m => m).length;
@@ -903,7 +907,9 @@ export class Explore extends React.PureComponent<ExploreProps, ExploreState> {
                         <Table data={tableResult} loading={tableLoading} onClickCell={this.onClickTableCell} />
                       </div>
                     ) : null}
-                    {supportsLogs && showingLogs ? <Logs data={logsResult} loading={logsLoading} /> : null}
+                    {supportsLogs && showingLogs ? (
+                      <Logs data={logsResult} loading={logsLoading} position={position} range={range} />
+                    ) : null}
                   </>
                 )}
               </ErrorBoundary>

public/app/features/explore/Graph.tsx

@@ -79,6 +79,7 @@ interface GraphProps {
   range: RawTimeRange;
   split?: boolean;
   size?: { width: number; height: number };
+  userOptions?: any;
 }
 
 interface GraphState {
@@ -122,7 +123,7 @@ export class Graph extends PureComponent<GraphProps, GraphState> {
   };
 
   draw() {
-    const { range, size } = this.props;
+    const { range, size, userOptions = {} } = this.props;
     const data = this.getGraphData();
 
     const $el = $(`#${this.props.id}`);
@@ -153,12 +154,14 @@ export class Graph extends PureComponent<GraphProps, GraphState> {
         max: max,
         label: 'Datetime',
         ticks: ticks,
+        timezone: 'browser',
         timeformat: time_format(ticks, min, max),
       },
     };
     const options = {
       ...FLOT_OPTIONS,
       ...dynamicOptions,
+      ...userOptions,
     };
     $.plot($el, series, options);
   }

public/app/features/explore/Logs.tsx

@@ -1,29 +1,130 @@
 import React, { Fragment, PureComponent } from 'react';
 import Highlighter from 'react-highlight-words';
 
+import { RawTimeRange } from 'app/types/series';
 import { LogsModel } from 'app/core/logs_model';
 import { findHighlightChunksInText } from 'app/core/utils/text';
+import { Switch } from 'app/core/components/Switch/Switch';
+
+import Graph from './Graph';
+
+const graphOptions = {
+  series: {
+    bars: {
+      show: true,
+    },
+  },
+  yaxis: {
+    tickDecimals: 0,
+  },
+};
 
 interface LogsProps {
   className?: string;
   data: LogsModel;
   loading: boolean;
+  position: string;
+  range?: RawTimeRange;
 }
 
-export default class Logs extends PureComponent<LogsProps, {}> {
+interface LogsState {
+  showLabels: boolean;
+  showLocalTime: boolean;
+  showUtc: boolean;
+}
+
+export default class Logs extends PureComponent<LogsProps, LogsState> {
+  state = {
+    showLabels: true,
+    showLocalTime: true,
+    showUtc: false,
+  };
+
+  onChangeLabels = (event: React.SyntheticEvent) => {
+    const target = event.target as HTMLInputElement;
+    this.setState({
+      showLabels: target.checked,
+    });
+  };
+
+  onChangeLocalTime = (event: React.SyntheticEvent) => {
+    const target = event.target as HTMLInputElement;
+    this.setState({
+      showLocalTime: target.checked,
+    });
+  };
+
+  onChangeUtc = (event: React.SyntheticEvent) => {
+    const target = event.target as HTMLInputElement;
+    this.setState({
+      showUtc: target.checked,
+    });
+  };
+
   render() {
-    const { className = '', data, loading = false } = this.props;
+    const { className = '', data, loading = false, position, range } = this.props;
+    const { showLabels, showLocalTime, showUtc } = this.state;
     const hasData = data && data.rows && data.rows.length > 0;
+    const cssColumnSizes = ['4px'];
+    if (showUtc) {
+      cssColumnSizes.push('minmax(100px, max-content)');
+    }
+    if (showLocalTime) {
+      cssColumnSizes.push('minmax(100px, max-content)');
+    }
+    if (showLabels) {
+      cssColumnSizes.push('minmax(100px, 25%)');
+    }
+    cssColumnSizes.push('1fr');
+    const logEntriesStyle = {
+      gridTemplateColumns: cssColumnSizes.join(' '),
+    };
+
     return (
       <div className={`${className} logs`}>
+        <div className="logs-graph">
+          <Graph
+            data={data.series}
+            height="100px"
+            range={range}
+            id={`explore-logs-graph-${position}`}
+            userOptions={graphOptions}
+          />
+        </div>
+
+        <div className="panel-container logs-options">
+          <div className="logs-controls">
+            <Switch label="Timestamp" checked={showUtc} onChange={this.onChangeUtc} small />
+            <Switch label="Local time" checked={showLocalTime} onChange={this.onChangeLocalTime} small />
+            <Switch label="Labels" checked={showLabels} onChange={this.onChangeLabels} small />
+            {hasData &&
+              data.meta && (
+                <div className="logs-meta">
+                  {data.meta.map(item => (
+                    <div className="logs-meta-item" key={item.label}>
+                      <span className="logs-meta-item__label">{item.label}:</span>
+                      <span className="logs-meta-item__value">{item.value}</span>
+                    </div>
+                  ))}
+                </div>
+              )}
+          </div>
+        </div>
+
         <div className="panel-container">
           {loading && <div className="explore-panel__loader" />}
-          <div className="logs-entries">
+          <div className="logs-entries" style={logEntriesStyle}>
             {hasData &&
               data.rows.map(row => (
                 <Fragment key={row.key}>
                   <div className={row.logLevel ? `logs-row-level logs-row-level-${row.logLevel}` : ''} />
-                  <div title={`${row.timestamp} (${row.timeFromNow})`}>{row.timeLocal}</div>
+                  {showUtc && <div title={`Local: ${row.timeLocal} (${row.timeFromNow})`}>{row.timestamp}</div>}
+                  {showLocalTime && <div title={`${row.timestamp} (${row.timeFromNow})`}>{row.timeLocal}</div>}
+                  {showLabels && (
+                    <div className="max-width" title={row.labels}>
+                      {row.labels}
+                    </div>
+                  )}
                   <div>
                     <Highlighter
                       textToHighlight={row.entry}

public/app/plugins/datasource/logging/datasource.ts

@@ -3,9 +3,10 @@ import _ from 'lodash';
 import * as dateMath from 'app/core/utils/datemath';
 
 import LanguageProvider from './language_provider';
-import { processStreams } from './result_transformer';
+import { mergeStreams, processStream } from './result_transformer';
+import { LogsStream } from 'app/core/logs_model';
 
-const DEFAULT_LIMIT = 100;
+const DEFAULT_LIMIT = 1000;
 
 const DEFAULT_QUERY_PARAMS = {
   direction: 'BACKWARD',
@@ -67,6 +68,10 @@ export default class LoggingDatasource {
     return this.backendSrv.datasourceRequest(req);
   }
 
+  mergeStreams(streams: LogsStream[]) {
+    return mergeStreams(streams, DEFAULT_LIMIT);
+  }
+
   prepareQueryTarget(target, options) {
     const interpolated = this.templateSrv.replace(target.expr);
     const start = this.getTime(options.range.from, false);
@@ -100,8 +105,8 @@ export default class LoggingDatasource {
         });
         return [...acc, ...streams];
       }, []);
-      const model = processStreams(allStreams, DEFAULT_LIMIT);
+      const processedStreams = allStreams.map(stream => processStream(stream, DEFAULT_LIMIT));
-      return { data: model };
+      return { data: processedStreams };
     });
   }
 

public/app/plugins/datasource/logging/result_transformer.test.ts

@@ -1,6 +1,6 @@
 import { LogLevel } from 'app/core/logs_model';
 
-import { getLogLevel } from './result_transformer';
+import { findCommonLabels, findUncommonLabels, formatLabels, getLogLevel, parseLabels } from './result_transformer';
 
 describe('getLoglevel()', () => {
   it('returns no log level on empty line', () => {
@@ -20,3 +20,57 @@ describe('getLoglevel()', () => {
     expect(getLogLevel('WARN this could be a debug message')).toBe(LogLevel.warn);
   });
 });
+
+describe('parseLabels()', () => {
+  it('returns no labels on emtpy labels string', () => {
+    expect(parseLabels('')).toEqual({});
+    expect(parseLabels('{}')).toEqual({});
+  });
+
+  it('returns labels on labels string', () => {
+    expect(parseLabels('{foo="bar", baz="42"}')).toEqual({ foo: '"bar"', baz: '"42"' });
+  });
+});
+
+describe('formatLabels()', () => {
+  it('returns no labels on emtpy label set', () => {
+    expect(formatLabels({})).toEqual('');
+    expect(formatLabels({}, 'foo')).toEqual('foo');
+  });
+
+  it('returns label string on label set', () => {
+    expect(formatLabels({ foo: '"bar"', baz: '"42"' })).toEqual('{baz="42", foo="bar"}');
+  });
+});
+
+describe('findCommonLabels()', () => {
+  it('returns no common labels on empty sets', () => {
+    expect(findCommonLabels([{}])).toEqual({});
+    expect(findCommonLabels([{}, {}])).toEqual({});
+  });
+
+  it('returns no common labels on differing sets', () => {
+    expect(findCommonLabels([{ foo: '"bar"' }, {}])).toEqual({});
+    expect(findCommonLabels([{}, { foo: '"bar"' }])).toEqual({});
+    expect(findCommonLabels([{ baz: '42' }, { foo: '"bar"' }])).toEqual({});
+    expect(findCommonLabels([{ foo: '42', baz: '"bar"' }, { foo: '"bar"' }])).toEqual({});
+  });
+
+  it('returns the single labels set as common labels', () => {
+    expect(findCommonLabels([{ foo: '"bar"' }])).toEqual({ foo: '"bar"' });
+  });
+});
+
+describe('findUncommonLabels()', () => {
+  it('returns no uncommon labels on empty sets', () => {
+    expect(findUncommonLabels({}, {})).toEqual({});
+  });
+
+  it('returns all labels given no common labels', () => {
+    expect(findUncommonLabels({ foo: '"bar"' }, {})).toEqual({ foo: '"bar"' });
+  });
+
+  it('returns all labels except the common labels', () => {
+    expect(findUncommonLabels({ foo: '"bar"', baz: '"42"' }, { foo: '"bar"' })).toEqual({ baz: '"42"' });
+  });
+});

public/app/plugins/datasource/logging/result_transformer.ts

@@ -1,7 +1,9 @@
 import _ from 'lodash';
 import moment from 'moment';
 
-import { LogLevel, LogsModel, LogRow } from 'app/core/logs_model';
+import { LogLevel, LogsMetaItem, LogsModel, LogRow, LogsStream } from 'app/core/logs_model';
+import { TimeSeries } from 'app/core/core';
+import colors from 'app/core/utils/colors';
 
 export function getLogLevel(line: string): LogLevel {
   if (!line) {
@@ -19,11 +21,65 @@ export function getLogLevel(line: string): LogLevel {
   return level;
 }
 
+const labelRegexp = /\b(\w+)(!?=~?)("[^"\n]*?")/g;
+export function parseLabels(labels: string): { [key: string]: string } {
+  const labelsByKey = {};
+  labels.replace(labelRegexp, (_, key, operator, value) => {
+    labelsByKey[key] = value;
+    return '';
+  });
+  return labelsByKey;
+}
+
+export function findCommonLabels(labelsSets: any[]) {
+  return labelsSets.reduce((acc, labels) => {
+    if (!labels) {
+      throw new Error('Need parsed labels to find common labels.');
+    }
+    if (!acc) {
+      // Initial set
+      acc = { ...labels };
+    } else {
+      // Remove incoming labels that are missing or not matching in value
+      Object.keys(labels).forEach(key => {
+        if (acc[key] === undefined || acc[key] !== labels[key]) {
+          delete acc[key];
+        }
+      });
+      // Remove common labels that are missing from incoming label set
+      Object.keys(acc).forEach(key => {
+        if (labels[key] === undefined) {
+          delete acc[key];
+        }
+      });
+    }
+    return acc;
+  }, undefined);
+}
+
+export function findUncommonLabels(labels, commonLabels) {
+  const uncommonLabels = { ...labels };
+  Object.keys(commonLabels).forEach(key => {
+    delete uncommonLabels[key];
+  });
+  return uncommonLabels;
+}
+
+export function formatLabels(labels, defaultValue = '') {
+  if (!labels || Object.keys(labels).length === 0) {
+    return defaultValue;
+  }
+  const labelKeys = Object.keys(labels).sort();
+  const cleanSelector = labelKeys.map(key => `${key}=${labels[key]}`).join(', ');
+  return ['{', cleanSelector, '}'].join('');
+}
+
 export function processEntry(entry: { line: string; timestamp: string }, stream): LogRow {
   const { line, timestamp } = entry;
   const { labels } = stream;
   const key = `EK${timestamp}${labels}`;
   const time = moment(timestamp);
+  const timeJs = time.valueOf();
   const timeFromNow = time.fromNow();
   const timeLocal = time.format('YYYY-MM-DD HH:mm:ss');
   const logLevel = getLogLevel(line);
@@ -32,21 +88,89 @@ export function processEntry(entry: { line: string; timestamp: string }, stream)
     key,
     logLevel,
     timeFromNow,
+    timeJs,
     timeLocal,
     entry: line,
+    labels: formatLabels(labels),
     searchWords: [stream.search],
     timestamp: timestamp,
   };
 }
 
-export function processStreams(streams, limit?: number): LogsModel {
+export function mergeStreams(streams: LogsStream[], limit?: number): LogsModel {
+  // Find meta data
+  const commonLabels = findCommonLabels(streams.map(stream => stream.parsedLabels));
+  const meta: LogsMetaItem[] = [
+    {
+      label: 'Common labels',
+      value: formatLabels(commonLabels),
+    },
+  ];
+
+  // Flatten entries of streams
   const combinedEntries = streams.reduce((acc, stream) => {
-    return [...acc, ...stream.entries.map(entry => processEntry(entry, stream))];
+    // Overwrite labels to be only the non-common ones
+    const labels = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels));
+    return [
+      ...acc,
+      ...stream.entries.map(entry => ({
+        ...entry,
+        labels,
+      })),
+    ];
   }, []);
+
+  const commonLabelsAlias =
+    streams.length === 1 ? formatLabels(commonLabels) : `Stream with common labels ${formatLabels(commonLabels)}`;
+  const series = streams.map((stream, index) => {
+    const colorIndex = index % colors.length;
+    stream.graphSeries.setColor(colors[colorIndex]);
+    stream.graphSeries.alias = formatLabels(findUncommonLabels(stream.parsedLabels, commonLabels), commonLabelsAlias);
+    return stream.graphSeries;
+  });
+
   const sortedEntries = _.chain(combinedEntries)
     .sortBy('timestamp')
     .reverse()
     .slice(0, limit || combinedEntries.length)
     .value();
-  return { rows: sortedEntries };
+
+  meta.push({
+    label: 'Limit',
+    value: `${limit} (${sortedEntries.length} returned)`,
+  });
+
+  return { meta, series, rows: sortedEntries };
+}
+
+export function processStream(stream: LogsStream, limit?: number): LogsStream {
+  const sortedEntries: any[] = _.chain(stream.entries)
+    .map(entry => processEntry(entry, stream))
+    .sortBy('timestamp')
+    .reverse()
+    .slice(0, limit || stream.entries.length)
+    .value();
+
+  // Build graph data
+  let previousTime;
+  const datapoints = sortedEntries.reduce((acc, entry, index) => {
+    // Bucket to nearest minute
+    const time = Math.round(entry.timeJs / 1000 / 60) * 1000 * 60;
+    // Entry for time
+    if (time === previousTime) {
+      acc[acc.length - 1][0]++;
+    } else {
+      acc.push([1, time]);
+      previousTime = time;
+    }
+    return acc;
+  }, []);
+  const graphSeries = new TimeSeries({ datapoints, alias: stream.labels });
+
+  return {
+    ...stream,
+    graphSeries,
+    entries: sortedEntries,
+    parsedLabels: parseLabels(stream.labels),
+  };
 }

public/sass/components/_gf-form.scss

@@ -116,6 +116,11 @@ $input-border: 1px solid $input-border-color;
     color: $critical;
   }
 
+  &--small {
+    padding: ($input-padding-y / 2) ($input-padding-x / 2);
+    font-size: $font-size-xs;
+  }
+
   &:disabled {
     color: $text-color-weak;
   }

public/sass/components/_switch.scss

@@ -41,7 +41,6 @@
     bottom: 0;
     right: 0;
     color: #fff;
-    font-size: $font-size-sm;
     text-align: center;
     font-size: 150%;
     display: flex;
@@ -91,6 +90,20 @@
     transform: rotateY(0);
   }
 
+  &--small {
+    max-width: 2rem;
+    min-width: 1.5rem;
+
+    input + label {
+      height: 25px;
+    }
+
+    input + label::before,
+    input + label::after {
+      font-size: $font-size-sm;
+    }
+  }
+
   &--table-cell {
     margin-bottom: 0;
     margin-right: 0;

public/sass/pages/_explore.scss

@@ -214,7 +214,42 @@
       display: grid;
       grid-column-gap: 1rem;
       grid-row-gap: 0.1rem;
-      grid-template-columns: 4px minmax(100px, max-content) 1fr;
+      grid-template-columns: 4px minmax(100px, max-content) minmax(100px, 25%) 1fr;
+      font-family: $font-family-monospace;
+      font-size: 12px;
+    }
+
+    .logs-controls {
+      display: flex;
+
+      > * {
+        margin-right: 1em;
+      }
+    }
+
+    .logs-options,
+    .logs-graph {
+      margin-bottom: $panel-margin;
+    }
+
+    .logs-meta {
+      flex: 1;
+      color: $text-color-weak;
+      padding: 2px 0;
+    }
+
+    .logs-meta-item {
+      display: inline-block;
+      margin-right: 1em;
+    }
+
+    .logs-meta-item__label {
+      margin-right: 0.5em;
+      font-size: 0.9em;
+      font-weight: 500;
+    }
+
+    .logs-meta-item__value {
       font-family: $font-family-monospace;
     }