Explore: Implement logs sample in Explore (#61864)

* Implement log samples

* Explore: Implement logs sample panel

* Log samples: Add documentation

* Update docs

* Add info for log sample

* Fix label

* Update

* Default to true

* Fix copy in test

* Update public/app/features/explore/LogsSamplePanel.tsx

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

* Use timeZone from grafana/schema

* Rename data props to queryResponse

* Unify name to logs sample

* Remove redundant optional parameters in LogsSamplePanel

* Make intervalMs parameter optional in dataFrameToLogsModel and remove undefined argument when not needed

* Fix incorrect position of copy log line button

* Update public/app/core/logsModel.ts

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

Co-authored-by: Sven Grossmann <sven.grossmann@grafana.com>

docs/sources/explore/logs-integration.md

@@ -141,6 +141,10 @@ after switching to the Logs data source, the query changes to:
 
 This will return a chunk of logs in the selected time range that can be grepped/text searched.
 
+### Logs sample
+
+If the selected data source implements logs sample, and supports both log and metric queries, then for metric queries you will be able to automatically see samples of log lines that contributed to visualized metrics. This feature is currently supported by Loki data sources.
+
 #### Live tailing
 
 Use the Live tailing feature to see real-time logs on supported data sources.

public/app/core/logsModel.ts

@@ -198,11 +198,13 @@ function isLogsData(series: DataFrame) {
  * Convert dataFrame into LogsModel which consists of creating separate array of log rows and metrics series. Metrics
  * series can be either already included in the dataFrame or will be computed from the log rows.
  * @param dataFrame
- * @param intervalMs In case there are no metrics series, we use this for computing it from log rows.
+ * @param intervalMs Optional. In case there are no metrics series, we use this for computing it from log rows.
+ * @param absoluteRange Optional. Used to store absolute range of executed queries in logs model. This is used for pagination.
+ * @param queries Optional. Used to store executed queries in logs model. This is used for pagination.
  */
 export function dataFrameToLogsModel(
   dataFrame: DataFrame[],
-  intervalMs: number | undefined,
+  intervalMs?: number,
   absoluteRange?: AbsoluteTimeRange,
   queries?: DataQuery[]
 ): LogsModel {
@@ -748,7 +750,7 @@ export function queryLogsVolume<TQuery extends DataQuery, TOptions extends DataS
 }
 
 /**
- * Creates an observable, which makes requests to get logs samples.
+ * Creates an observable, which makes requests to get logs sample.
  */
 export function queryLogsSample<TQuery extends DataQuery, TOptions extends DataSourceJsonData>(
   datasource: DataSourceApi<TQuery, TOptions>,

public/app/features/explore/Explore.test.tsx

@@ -87,6 +87,9 @@ const dummyProps: Props = {
   isFromCompactUrl: false,
   eventBus: new EventBusSrv(),
   showRawPrometheus: false,
+  showLogsSample: false,
+  logsSample: { enabled: false },
+  setSupplementaryQueryEnabled: jest.fn(),
 };
 
 jest.mock('@grafana/runtime/src/services/dataSourceSrv', () => {

public/app/features/explore/Explore.tsx

@@ -15,6 +15,7 @@ import {
   RawTimeRange,
   EventBus,
   SplitOpenOptions,
+  SupplementaryQueryType,
 } from '@grafana/data';
 import { selectors } from '@grafana/e2e-selectors';
 import { config, getDataSourceSrv, reportInteraction } from '@grafana/runtime';
@@ -44,6 +45,7 @@ import { ExploreToolbar } from './ExploreToolbar';
 import { FlameGraphExploreContainer } from './FlameGraphExploreContainer';
 import { GraphContainer } from './Graph/GraphContainer';
 import LogsContainer from './LogsContainer';
+import { LogsSamplePanel } from './LogsSamplePanel';
 import { NoData } from './NoData';
 import { NoDataSourceCallToAction } from './NoDataSourceCallToAction';
 import { NodeGraphContainer } from './NodeGraphContainer';
@@ -56,7 +58,14 @@ import TableContainer from './TableContainer';
 import { TraceViewContainer } from './TraceView/TraceViewContainer';
 import { changeSize } from './state/explorePane';
 import { splitOpen } from './state/main';
-import { addQueryRow, modifyQueries, scanStart, scanStopAction, setQueries } from './state/query';
+import {
+  addQueryRow,
+  modifyQueries,
+  scanStart,
+  scanStopAction,
+  setQueries,
+  setSupplementaryQueryEnabled,
+} from './state/query';
 import { isSplit } from './state/selectors';
 import { makeAbsoluteTime, updateTimeRange } from './state/time';
 
@@ -357,6 +366,22 @@ export class Explore extends React.PureComponent<Props, ExploreState> {
     );
   }
 
+  renderLogsSamplePanel() {
+    const { logsSample, timeZone, setSupplementaryQueryEnabled, exploreId, datasourceInstance } = this.props;
+
+    return (
+      <LogsSamplePanel
+        queryResponse={logsSample.data}
+        timeZone={timeZone}
+        enabled={logsSample.enabled}
+        datasourceInstance={datasourceInstance}
+        setLogsSampleEnabled={(enabled) =>
+          setSupplementaryQueryEnabled(exploreId, enabled, SupplementaryQueryType.LogsSample)
+        }
+      />
+    );
+  }
+
   renderNodeGraphPanel() {
     const { exploreId, showTrace, queryResponse, datasourceInstance } = this.props;
     const datasourceType = datasourceInstance ? datasourceInstance?.type : 'unknown';
@@ -416,6 +441,7 @@ export class Explore extends React.PureComponent<Props, ExploreState> {
       showFlameGraph,
       timeZone,
       isFromCompactUrl,
+      showLogsSample,
     } = this.props;
     const { openDrawer } = this.state;
     const styles = getStyles(theme);
@@ -486,6 +512,7 @@ export class Explore extends React.PureComponent<Props, ExploreState> {
                             <ErrorBoundaryAlert>{this.renderFlameGraphPanel()}</ErrorBoundaryAlert>
                           )}
                           {showTrace && <ErrorBoundaryAlert>{this.renderTraceViewPanel()}</ErrorBoundaryAlert>}
+                          {showLogsSample && <ErrorBoundaryAlert>{this.renderLogsSamplePanel()}</ErrorBoundaryAlert>}
                           {showNoData && <ErrorBoundaryAlert>{this.renderNoData()}</ErrorBoundaryAlert>}
                         </>
                       )}
@@ -528,6 +555,7 @@ function mapStateToProps(state: StoreState, { exploreId }: ExploreProps) {
     queries,
     isLive,
     graphResult,
+    tableResult,
     logsResult,
     showLogs,
     showMetrics,
@@ -540,8 +568,13 @@ function mapStateToProps(state: StoreState, { exploreId }: ExploreProps) {
     loading,
     isFromCompactUrl,
     showRawPrometheus,
+    supplementaryQueries,
   } = item;
 
+  const logsSample = supplementaryQueries[SupplementaryQueryType.LogsSample];
+  // We want to show logs sample only if there are no log results and if there is already graph or table result
+  const showLogsSample = !!(logsSample.dataProvider !== undefined && !logsResult && (graphResult || tableResult));
+
   return {
     datasourceInstance,
     datasourceMissing,
@@ -564,6 +597,8 @@ function mapStateToProps(state: StoreState, { exploreId }: ExploreProps) {
     splitted: isSplit(state),
     loading,
     isFromCompactUrl: isFromCompactUrl || false,
+    logsSample,
+    showLogsSample,
   };
 }
 
@@ -577,6 +612,7 @@ const mapDispatchToProps = {
   makeAbsoluteTime,
   addQueryRow,
   splitOpen,
+  setSupplementaryQueryEnabled,
 };
 
 const connector = connect(mapStateToProps, mapDispatchToProps);

public/app/features/explore/LogsSample.test.tsx

@@ -0,0 +1,103 @@
+import { render, screen } from '@testing-library/react';
+import userEvent from '@testing-library/user-event';
+import React, { ComponentProps } from 'react';
+
+import { ArrayVector, FieldType, LoadingState, MutableDataFrame } from '@grafana/data';
+
+import { LogsSamplePanel } from './LogsSamplePanel';
+
+jest.mock('@grafana/runtime', () => {
+  return {
+    ...jest.requireActual('@grafana/runtime'),
+    reportInteraction: jest.fn(),
+  };
+});
+
+const createProps = (propOverrides?: Partial<ComponentProps<typeof LogsSamplePanel>>) => {
+  const props = {
+    queryResponse: undefined,
+    enabled: true,
+    timeZone: 'timeZone',
+    datasourceInstance: undefined,
+    setLogsSampleEnabled: jest.fn(),
+  };
+
+  return { ...props, ...propOverrides };
+};
+
+const sampleDataFrame = new MutableDataFrame({
+  meta: {
+    custom: { frameType: 'LabeledTimeValues' },
+  },
+  fields: [
+    {
+      name: 'labels',
+      type: FieldType.other,
+      values: new ArrayVector([
+        { place: 'luna', source: 'data' },
+        { place: 'luna', source: 'data' },
+      ]),
+    },
+    {
+      name: 'Time',
+      type: FieldType.time,
+      values: new ArrayVector(['2022-02-22T09:28:11.352440161Z', '2022-02-22T14:42:50.991981292Z']),
+    },
+    {
+      name: 'Line',
+      type: FieldType.string,
+      values: new ArrayVector(['line1 ', 'line2']),
+    },
+  ],
+});
+
+describe('LogsSamplePanel', () => {
+  it('shows empty panel if no data', () => {
+    render(<LogsSamplePanel {...createProps()} />);
+    expect(screen.getByText('Logs sample')).toBeInTheDocument();
+  });
+
+  it('shows loading message', () => {
+    render(<LogsSamplePanel {...createProps({ queryResponse: { data: [], state: LoadingState.Loading } })} />);
+    expect(screen.getByText('Logs sample is loading...')).toBeInTheDocument();
+  });
+
+  it('shows no data message', () => {
+    render(<LogsSamplePanel {...createProps({ queryResponse: { data: [], state: LoadingState.Done } })} />);
+    expect(screen.getByText('No logs sample data.')).toBeInTheDocument();
+  });
+
+  it('shows logs sample data', () => {
+    render(
+      <LogsSamplePanel {...createProps({ queryResponse: { data: [sampleDataFrame], state: LoadingState.Done } })} />
+    );
+    expect(screen.getByText('2022-02-22 04:28:11')).toBeInTheDocument();
+    expect(screen.getByText('line1')).toBeInTheDocument();
+    expect(screen.getByText('2022-02-22 09:42:50')).toBeInTheDocument();
+    expect(screen.getByText('line2')).toBeInTheDocument();
+  });
+
+  it('shows log details', async () => {
+    render(
+      <LogsSamplePanel {...createProps({ queryResponse: { data: [sampleDataFrame], state: LoadingState.Done } })} />
+    );
+    const line = screen.getByText('line1');
+    expect(screen.queryByText('foo')).not.toBeInTheDocument();
+    await userEvent.click(line);
+    expect(await screen.findByText('Fields')).toBeInTheDocument();
+    expect(await screen.findByText('place')).toBeInTheDocument();
+    expect(await screen.findByText('luna')).toBeInTheDocument();
+  });
+
+  it('shows warning message', () => {
+    render(
+      <LogsSamplePanel
+        {...createProps({
+          queryResponse: { data: [], state: LoadingState.Error, error: { data: { message: 'Test error message' } } },
+        })}
+      />
+    );
+    expect(screen.getByText('Failed to load logs sample for this query')).toBeInTheDocument();
+    expect(screen.getByText('Test error message')).toBeInTheDocument();
+  });
+});

public/app/features/explore/LogsSamplePanel.tsx

@@ -0,0 +1,67 @@
+import React from 'react';
+
+import { DataQueryResponse, DataSourceApi, LoadingState, LogsDedupStrategy } from '@grafana/data';
+import { reportInteraction } from '@grafana/runtime';
+import { TimeZone } from '@grafana/schema';
+import { Collapse } from '@grafana/ui';
+import { dataFrameToLogsModel } from 'app/core/logsModel';
+import store from 'app/core/store';
+
+import { LogRows } from '../logs/components/LogRows';
+
+import { SupplementaryResultError } from './SupplementaryResultError';
+import { SETTINGS_KEYS } from './utils/logs';
+
+type Props = {
+  queryResponse: DataQueryResponse | undefined;
+  enabled: boolean;
+  timeZone: TimeZone;
+  datasourceInstance: DataSourceApi | null | undefined;
+  setLogsSampleEnabled: (enabled: boolean) => void;
+};
+
+export function LogsSamplePanel(props: Props) {
+  const { queryResponse, timeZone, enabled, setLogsSampleEnabled, datasourceInstance } = props;
+
+  const onToggleLogsSampleCollapse = (isOpen: boolean) => {
+    setLogsSampleEnabled(isOpen);
+    reportInteraction('grafana_explore_logs_sample_toggle_clicked', {
+      datasourceType: datasourceInstance?.type ?? 'unknown',
+      type: isOpen ? 'open' : 'close',
+    });
+  };
+
+  let LogsSamplePanelContent: JSX.Element | null;
+
+  if (queryResponse === undefined) {
+    LogsSamplePanelContent = null;
+  } else if (queryResponse.error !== undefined) {
+    LogsSamplePanelContent = (
+      <SupplementaryResultError error={queryResponse.error} title="Failed to load logs sample for this query" />
+    );
+  } else if (queryResponse.state === LoadingState.Loading) {
+    LogsSamplePanelContent = <span>Logs sample is loading...</span>;
+  } else if (queryResponse.data.length === 0 || queryResponse.data[0].length === 0) {
+    LogsSamplePanelContent = <span>No logs sample data.</span>;
+  } else {
+    const logs = dataFrameToLogsModel(queryResponse.data);
+    LogsSamplePanelContent = (
+      <LogRows
+        logRows={logs.rows}
+        dedupStrategy={LogsDedupStrategy.none}
+        showLabels={store.getBool(SETTINGS_KEYS.showLabels, false)}
+        showTime={store.getBool(SETTINGS_KEYS.showTime, true)}
+        wrapLogMessage={store.getBool(SETTINGS_KEYS.wrapLogMessage, true)}
+        prettifyLogMessage={store.getBool(SETTINGS_KEYS.prettifyLogMessage, false)}
+        timeZone={timeZone}
+        enableLogDetails={true}
+      />
+    );
+  }
+
+  return (
+    <Collapse label="Logs sample" isOpen={enabled} collapsible={true} onToggle={onToggleLogsSampleCollapse}>
+      {LogsSamplePanelContent}
+    </Collapse>
+  );
+}

public/app/features/explore/LogsVolumePanel.tsx

@@ -1,9 +1,8 @@
 import { css } from '@emotion/css';
-import React, { useState } from 'react';
+import React from 'react';
 
 import {
   AbsoluteTimeRange,
-  DataQueryError,
   DataQueryResponse,
   GrafanaTheme2,
   LoadingState,
@@ -11,9 +10,10 @@ import {
   TimeZone,
   EventBus,
 } from '@grafana/data';
-import { Alert, Button, Collapse, InlineField, TooltipDisplayMode, useStyles2, useTheme2 } from '@grafana/ui';
+import { Button, Collapse, InlineField, TooltipDisplayMode, useStyles2, useTheme2 } from '@grafana/ui';
 
 import { ExploreGraph } from './Graph/ExploreGraph';
+import { SupplementaryResultError } from './SupplementaryResultError';
 
 type Props = {
   logsVolumeData: DataQueryResponse | undefined;
@@ -29,35 +29,6 @@ type Props = {
   eventBus: EventBus;
 };
 
-const SHORT_ERROR_MESSAGE_LIMIT = 100;
-
-function ErrorAlert(props: { error: DataQueryError }) {
-  const [isOpen, setIsOpen] = useState(false);
-  // generic get-error-message-logic, taken from
-  // /public/app/features/explore/ErrorContainer.tsx
-  const message = props.error.message || props.error.data?.message || '';
-
-  const showButton = !isOpen && message.length > SHORT_ERROR_MESSAGE_LIMIT;
-
-  return (
-    <Alert title="Failed to load log volume for this query" severity="warning">
-      {showButton ? (
-        <Button
-          variant="secondary"
-          size="xs"
-          onClick={() => {
-            setIsOpen(true);
-          }}
-        >
-          Show details
-        </Button>
-      ) : (
-        message
-      )}
-    </Alert>
-  );
-}
-
 function createVisualisationData(
   logLinesBased: DataQueryResponse | undefined,
   logLinesBasedVisibleRange: AbsoluteTimeRange | undefined,
@@ -110,7 +81,7 @@ export function LogsVolumePanel(props: Props) {
   const { logsVolumeData, fullRangeData, range } = data;
 
   if (logsVolumeData.error !== undefined) {
-    return <ErrorAlert error={logsVolumeData.error} />;
+    return <SupplementaryResultError error={logsVolumeData.error} title="Failed to load log volume for this query" />;
   }
 
   let LogsVolumePanelContent;

public/app/features/explore/SupplementaryResultError.test.tsx

@@ -0,0 +1,30 @@
+import { render, screen } from '@testing-library/react';
+import React from 'react';
+
+import { SupplementaryResultError } from './SupplementaryResultError';
+
+describe('SupplementaryResultError', () => {
+  it('shows short warning message', () => {
+    const error = { data: { message: 'Test error message' } };
+    const title = 'Error loading supplementary query';
+
+    render(<SupplementaryResultError error={error} title={title} />);
+    expect(screen.getByText(title)).toBeInTheDocument();
+    expect(screen.getByText(error.data.message)).toBeInTheDocument();
+  });
+
+  it('shows long warning message', () => {
+    // we make a long message
+    const messagePart = 'One two three four five six seven eight nine ten.';
+    const message = messagePart.repeat(3);
+    const error = { data: { message } };
+    const title = 'Error loading supplementary query';
+
+    render(<SupplementaryResultError error={error} title={title} />);
+    expect(screen.getByText(title)).toBeInTheDocument();
+    expect(screen.queryByText(message)).not.toBeInTheDocument();
+    const button = screen.getByText('Show details');
+    button.click();
+    expect(screen.getByText(message)).toBeInTheDocument();
+  });
+});

public/app/features/explore/SupplementaryResultError.tsx

@@ -0,0 +1,36 @@
+import React, { useState } from 'react';
+
+import { DataQueryError } from '@grafana/data';
+import { Alert, Button } from '@grafana/ui';
+
+type Props = {
+  error: DataQueryError;
+  title: string;
+};
+export function SupplementaryResultError(props: Props) {
+  const [isOpen, setIsOpen] = useState(false);
+  const SHORT_ERROR_MESSAGE_LIMIT = 100;
+  const { error, title } = props;
+  // generic get-error-message-logic, taken from
+  // /public/app/features/explore/ErrorContainer.tsx
+  const message = error.message || error.data?.message || '';
+  const showButton = !isOpen && message.length > SHORT_ERROR_MESSAGE_LIMIT;
+
+  return (
+    <Alert title={title} severity="warning">
+      {showButton ? (
+        <Button
+          variant="secondary"
+          size="xs"
+          onClick={() => {
+            setIsOpen(true);
+          }}
+        >
+          Show details
+        </Button>
+      ) : (
+        message
+      )}
+    </Alert>
+  );
+}

public/app/features/explore/state/query.test.ts

@@ -517,7 +517,7 @@ describe('reducer', () => {
       mockDataProvider = () => {
         return of({ state: LoadingState.Done, error: undefined, data: [{}] });
       };
-      // turn logs volume off (but keep log sample on)
+      // turn logs volume off (but keep logs sample on)
       dispatch(setSupplementaryQueryEnabled(ExploreId.left, false, SupplementaryQueryType.LogsVolume));
       expect(getState().explore[ExploreId.left].supplementaryQueries[SupplementaryQueryType.LogsVolume].enabled).toBe(
         false

public/app/features/explore/utils/supplementaryQueries.ts

@@ -17,8 +17,7 @@ export const loadSupplementaryQueries = (): SupplementaryQueries => {
   // We default to true for all supp queries
   let supplementaryQueries: SupplementaryQueries = {
     [SupplementaryQueryType.LogsVolume]: { enabled: true },
-    // This is set to false temporarily, until we have UI to display logs sample and a way how to enable/disable it
-    [SupplementaryQueryType.LogsSample]: { enabled: false },
+    [SupplementaryQueryType.LogsSample]: { enabled: true },
   };
 
   for (const type of supplementaryQueryTypes) {

public/app/features/inspector/InspectDataTab.tsx

@@ -108,7 +108,7 @@ export class InspectDataTab extends PureComponent<Props, State> {
       area: 'inspector',
     });
 
-    const logsModel = dataFrameToLogsModel(data || [], undefined);
+    const logsModel = dataFrameToLogsModel(data || []);
     downloadLogsModelAsTxt(logsModel, panel ? panel.getDisplayTitle() : 'Explore');
   };
 

public/app/features/logs/components/LogRowMessage.tsx

@@ -32,7 +32,7 @@ interface Props extends Themeable2 {
   logsSortOrder?: LogsSortOrder | null;
 }
 
-const getStyles = (theme: GrafanaTheme2, showContextButton: boolean, isInDashboard: boolean | undefined) => {
+const getStyles = (theme: GrafanaTheme2, showContextButton: boolean, isInExplore: boolean) => {
   const outlineColor = tinycolor(theme.components.dashboard.background).setAlpha(0.7).toRgbString();
 
   return {
@@ -74,7 +74,7 @@ const getStyles = (theme: GrafanaTheme2, showContextButton: boolean, isInDashboa
     `,
     logRowMenuCell: css`
       position: absolute;
-      right: ${isInDashboard ? '40px' : `calc(75px + ${theme.spacing()} + ${showContextButton ? '80px' : '40px'})`};
+      right: ${!isInExplore ? '40px' : `calc(75px + ${theme.spacing()} + ${showContextButton ? '80px' : '40px'})`};
       margin-top: -${theme.spacing(0.125)};
     `,
     logLine: css`
@@ -169,7 +169,7 @@ class UnThemedLogRowMessage extends PureComponent<Props> {
     const { hasAnsi, raw } = row;
     const restructuredEntry = restructureLog(raw, prettifyLogMessage);
     const shouldShowContextToggle = showContextToggle ? showContextToggle(row) : false;
-    const styles = getStyles(theme, shouldShowContextToggle, app === CoreApp.Dashboard);
+    const styles = getStyles(theme, shouldShowContextToggle, app === CoreApp.Explore);
 
     return (
       <>

public/app/plugins/datasource/loki/datasource.ts

@@ -471,7 +471,7 @@ export class LokiDatasource
   }
 
   async getDataSamples(query: LokiQuery): Promise<DataFrame[]> {
-    // Currently works only for log samples
+    // Currently works only for logs sample
     if (!isValidQuery(query.expr) || !isLogsQuery(query.expr)) {
       return [];
     }