Tempo: Separate trace to logs and loki search datasource config (#46655)

* Tempo: Separate trace to logs and loki search datasource config

* Add getLokiSearchDS util and tests

docs/sources/datasources/jaeger.md

@@ -156,7 +156,6 @@ datasources:
         spanEndTimeShift: '1h'
         filterByTraceID: false
         filterBySpanID: false
-        lokiSearch: true
     secureJsonData:
       basicAuthPassword: my_password
 ```

docs/sources/datasources/tempo.md

@@ -57,6 +57,12 @@ This is a configuration for the beta Node Graph visualization. The Node Graph is
 
 -- **Enable Node Graph -** Enables the Node Graph visualization.
 
+### Loki Search
+
+This is a configuration for the Loki search query type.
+
+-- **Data source -** The Loki instance in which you want to search traces. You must configure derived fields in the Loki instance.
+
 ## Query traces
 
 You can query and display traces from Tempo via [Explore]({{< relref "../explore/_index.md" >}}).
@@ -81,7 +87,7 @@ You must also configure your Tempo data source to use this feature.Refer to the
 
 ### Loki search
 
-You can search for traces if you set up the trace to logs setting in the data source configuration page. To find traces to visualize, use the [Loki query editor]({{< relref "loki.md#loki-query-editor" >}}). To get search results, you must have [derived fields]({{< relref "loki.md#derived-fields" >}}) configured, which point to this data source.
+To find traces to visualize, use the [Loki query editor]({{< relref "loki.md#loki-query-editor" >}}). To get search results, you must have [derived fields]({{< relref "loki.md#derived-fields" >}}) configured, which point to this data source.
 
 {{< figure src="/static/img/docs/tempo/query-editor-search.png" class="docs-image--no-shadow" max-width="750px" caption="Screenshot of the Tempo query editor showing the search tab" >}}
 
@@ -197,11 +203,12 @@ datasources:
         spanEndTimeShift: '1h'
         filterByTraceID: false
         filterBySpanID: false
-        lokiSearch: true
       serviceMap:
         datasourceUid: 'prometheus'
       search:
         hide: false
       nodeGraph:
         enabled: true
+      lokiSearch:
+        datasourceUid: 'loki'
 ```

public/app/core/components/TraceToLogs/TraceToLogsSettings.tsx

@@ -20,7 +20,7 @@ export interface TraceToLogsOptions {
   spanEndTimeShift?: string;
   filterByTraceID?: boolean;
   filterBySpanID?: boolean;
-  lokiSearch?: boolean;
+  lokiSearch?: boolean; // legacy
 }
 
 export interface TraceToLogsData extends DataSourceJsonData {
@@ -205,22 +205,6 @@ export function TraceToLogsSettings({ options, onOptionsChange }: Props) {
           />
         </InlineField>
       </InlineFieldRow>
-
-      <InlineFieldRow>
-        <InlineField label="Loki Search" labelWidth={26} grow tooltip="Use this logs data source to search for traces.">
-          <InlineSwitch
-            id="lokiSearch"
-            defaultChecked={true}
-            value={options.jsonData.tracesToLogs?.lokiSearch}
-            onChange={(event: React.SyntheticEvent<HTMLInputElement>) =>
-              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToLogs', {
-                ...options.jsonData.tracesToLogs,
-                lokiSearch: event.currentTarget.checked,
-              })
-            }
-          />
-        </InlineField>
-      </InlineFieldRow>
     </div>
   );
 }

public/app/plugins/datasource/tempo/QueryEditor/QueryField.tsx

@@ -12,13 +12,11 @@ import {
   Themeable2,
   withTheme2,
 } from '@grafana/ui';
-import { TraceToLogsOptions } from 'app/core/components/TraceToLogs/TraceToLogsSettings';
 import React from 'react';
 import { LokiQueryField } from '../../loki/components/LokiQueryField';
 import { LokiQuery } from '../../loki/types';
 import { TempoDatasource, TempoQuery, TempoQueryType } from '../datasource';
 import LokiDatasource from '../../loki/datasource';
-import { PrometheusDatasource } from '../../prometheus/datasource';
 import useAsync from 'react-use/lib/useAsync';
 import NativeSearch from './NativeSearch';
 import { getDS } from './utils';
@@ -28,43 +26,12 @@ interface Props extends QueryEditorProps<TempoDatasource, TempoQuery>, Themeable
 
 const DEFAULT_QUERY_TYPE: TempoQueryType = 'traceId';
 
-interface State {
-  linkedDatasourceUid?: string;
-  linkedDatasource?: LokiDatasource;
-  serviceMapDatasourceUid?: string;
-  serviceMapDatasource?: PrometheusDatasource;
-}
-
-class TempoQueryFieldComponent extends React.PureComponent<Props, State> {
-  state = {
-    linkedDatasourceUid: undefined,
-    linkedDatasource: undefined,
-    serviceMapDatasourceUid: undefined,
-    serviceMapDatasource: undefined,
-  };
-
+class TempoQueryFieldComponent extends React.PureComponent<Props> {
   constructor(props: Props) {
     super(props);
   }
 
   async componentDidMount() {
-    const { datasource } = this.props;
-    // Find query field from linked datasource
-    const tracesToLogsOptions: TraceToLogsOptions = datasource.tracesToLogs || {};
-    const linkedDatasourceUid = tracesToLogsOptions.datasourceUid;
-
-    const serviceMapDsUid = datasource.serviceMap?.datasourceUid;
-
-    // Check status of linked data sources so we can show warnings if needed.
-    const [logsDs, serviceMapDs] = await Promise.all([getDS(linkedDatasourceUid), getDS(serviceMapDsUid)]);
-
-    this.setState({
-      linkedDatasourceUid: linkedDatasourceUid,
-      linkedDatasource: logsDs as LokiDatasource,
-      serviceMapDatasourceUid: serviceMapDsUid,
-      serviceMapDatasource: serviceMapDs as PrometheusDatasource,
-    });
-
     // Set initial query type to ensure traceID field appears
     if (!this.props.query.queryType) {
       this.props.onChange({
@@ -98,9 +65,9 @@ class TempoQueryFieldComponent extends React.PureComponent<Props, State> {
 
   render() {
     const { query, onChange, datasource } = this.props;
-    // Find query field from linked datasource
-    const tracesToLogsOptions: TraceToLogsOptions = datasource.tracesToLogs || {};
-    const logsDatasourceUid = tracesToLogsOptions.datasourceUid;
+
+    const logsDatasourceUid = datasource.getLokiSearchDS();
+
     const graphDatasourceUid = datasource.serviceMap?.datasourceUid;
 
     const queryTypeOptions: Array<SelectableValue<TempoQueryType>> = [
@@ -116,7 +83,7 @@ class TempoQueryFieldComponent extends React.PureComponent<Props, State> {
       queryTypeOptions.unshift({ value: 'nativeSearch', label: 'Search - Beta' });
     }
 
-    if (logsDatasourceUid && tracesToLogsOptions?.lokiSearch !== false) {
+    if (logsDatasourceUid) {
       if (!config.featureToggles.tempoSearch) {
         // Place at beginning as Search if no native search
         queryTypeOptions.unshift({ value: 'search', label: 'Search' });
@@ -161,7 +128,7 @@ class TempoQueryFieldComponent extends React.PureComponent<Props, State> {
         )}
         {query.queryType === 'search' && (
           <SearchSection
-            linkedDatasourceUid={logsDatasourceUid}
+            logsDatasourceUid={logsDatasourceUid}
             query={query}
             onRunQuery={this.onRunLinkedQuery}
             onChange={this.onChangeLinkedQuery}
@@ -217,13 +184,13 @@ class TempoQueryFieldComponent extends React.PureComponent<Props, State> {
 }
 
 interface SearchSectionProps {
-  linkedDatasourceUid?: string;
+  logsDatasourceUid?: string;
   onChange: (value: LokiQuery) => void;
   onRunQuery: () => void;
   query: TempoQuery;
 }
-function SearchSection({ linkedDatasourceUid, onChange, onRunQuery, query }: SearchSectionProps) {
-  const dsState = useAsync(() => getDS(linkedDatasourceUid), [linkedDatasourceUid]);
+function SearchSection({ logsDatasourceUid, onChange, onRunQuery, query }: SearchSectionProps) {
+  const dsState = useAsync(() => getDS(logsDatasourceUid), [logsDatasourceUid]);
   if (dsState.loading) {
     return null;
   }
@@ -246,15 +213,15 @@ function SearchSection({ linkedDatasourceUid, onChange, onRunQuery, query }: Sea
     );
   }
 
-  if (!linkedDatasourceUid) {
-    return <div className="text-warning">Please set up a Traces-to-logs datasource in the datasource settings.</div>;
+  if (!logsDatasourceUid) {
+    return <div className="text-warning">Please set up a Loki search datasource in the datasource settings.</div>;
   }
 
-  if (linkedDatasourceUid && !ds) {
+  if (logsDatasourceUid && !ds) {
     return (
       <div className="text-warning">
-        Traces-to-logs datasource is configured but the data source no longer exists. Please configure existing data
-        source to use the search.
+        Loki search datasource is configured but the data source no longer exists. Please configure existing data source
+        to use the search.
       </div>
     );
   }

public/app/plugins/datasource/tempo/configuration/ConfigEditor.tsx

@@ -6,6 +6,7 @@ import { ServiceGraphSettings } from './ServiceGraphSettings';
 import { config } from '@grafana/runtime';
 import { SearchSettings } from './SearchSettings';
 import { NodeGraphSettings } from 'app/core/components/NodeGraphSettings';
+import { LokiSearchSettings } from './LokiSearchSettings';
 
 export type Props = DataSourcePluginOptionsEditorProps;
 
@@ -35,6 +36,9 @@ export const ConfigEditor: React.FC<Props> = ({ options, onOptionsChange }) => {
       <div className="gf-form-group">
         <NodeGraphSettings options={options} onOptionsChange={onOptionsChange} />
       </div>
+      <div className="gf-form-group">
+        <LokiSearchSettings options={options} onOptionsChange={onOptionsChange} />
+      </div>
     </>
   );
 };

public/app/plugins/datasource/tempo/configuration/LokiSearchSettings.tsx

@@ -0,0 +1,77 @@
+import { css } from '@emotion/css';
+import { DataSourcePluginOptionsEditorProps, GrafanaTheme, updateDatasourcePluginJsonDataOption } from '@grafana/data';
+import { DataSourcePicker } from '@grafana/runtime';
+import { Button, InlineField, InlineFieldRow, useStyles } from '@grafana/ui';
+import React from 'react';
+import { TempoJsonData } from '../datasource';
+
+interface Props extends DataSourcePluginOptionsEditorProps<TempoJsonData> {}
+
+export function LokiSearchSettings({ options, onOptionsChange }: Props) {
+  const styles = useStyles(getStyles);
+
+  // Default to the trace to logs datasource if configured and loki search was enabled
+  // but only if jsonData.lokiSearch hasn't been set
+  const legacyDatasource =
+    options.jsonData.tracesToLogs?.lokiSearch !== false ? options.jsonData.tracesToLogs?.datasourceUid : undefined;
+  if (legacyDatasource && options.jsonData.lokiSearch === undefined) {
+    updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'lokiSearch', {
+      datasourceUid: legacyDatasource,
+    });
+  }
+
+  return (
+    <div className={css({ width: '100%' })}>
+      <h3 className="page-heading">Loki Search</h3>
+
+      <div className={styles.infoText}>
+        Select a Loki datasource to search for traces. Derived fields must be configured in the Loki data source.
+      </div>
+
+      <InlineFieldRow className={styles.row}>
+        <InlineField tooltip="The Loki data source with the service graph data" label="Data source" labelWidth={26}>
+          <DataSourcePicker
+            inputId="loki-search-data-source-picker"
+            pluginId="loki"
+            current={options.jsonData.lokiSearch?.datasourceUid}
+            noDefault={true}
+            width={40}
+            onChange={(ds) =>
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'lokiSearch', {
+                datasourceUid: ds.uid,
+              })
+            }
+          />
+        </InlineField>
+        {options.jsonData.lokiSearch?.datasourceUid ? (
+          <Button
+            type={'button'}
+            variant={'secondary'}
+            size={'sm'}
+            fill={'text'}
+            onClick={() => {
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'lokiSearch', {
+                datasourceUid: undefined,
+              });
+            }}
+          >
+            Clear
+          </Button>
+        ) : null}
+      </InlineFieldRow>
+    </div>
+  );
+}
+
+const getStyles = (theme: GrafanaTheme) => ({
+  infoText: css`
+    label: infoText;
+    padding-bottom: ${theme.spacing.md};
+    color: ${theme.colors.textSemiWeak};
+  `,
+
+  row: css`
+    label: row;
+    align-items: baseline;
+  `,
+});

public/app/plugins/datasource/tempo/datasource.test.ts

@@ -223,6 +223,55 @@ describe('Tempo data source', () => {
       'Service Name: frontend, Span Name: /config, Search: root.http.status_code=500, Min Duration: 1ms, Max Duration: 100s, Limit: 10'
     );
   });
+
+  it('should get loki search datasource', () => {
+    // 1. Get lokiSearch.datasource if present
+    const ds1 = new TempoDatasource({
+      ...defaultSettings,
+      jsonData: {
+        lokiSearch: {
+          datasourceUid: 'loki-1',
+        },
+      },
+    });
+    const lokiDS1 = ds1.getLokiSearchDS();
+    expect(lokiDS1).toBe('loki-1');
+
+    // 2. Get traceToLogs.datasource
+    const ds2 = new TempoDatasource({
+      ...defaultSettings,
+      jsonData: {
+        tracesToLogs: {
+          lokiSearch: true,
+          datasourceUid: 'loki-2',
+        },
+      },
+    });
+    const lokiDS2 = ds2.getLokiSearchDS();
+    expect(lokiDS2).toBe('loki-2');
+
+    // 3. Return undefined if neither is available
+    const ds3 = new TempoDatasource(defaultSettings);
+    const lokiDS3 = ds3.getLokiSearchDS();
+    expect(lokiDS3).toBe(undefined);
+
+    // 4. Return undefined if lokiSearch is undefined, even if traceToLogs is present
+    // since this indicates the user cleared the fallback setting
+    const ds4 = new TempoDatasource({
+      ...defaultSettings,
+      jsonData: {
+        tracesToLogs: {
+          lokiSearch: true,
+          datasourceUid: 'loki-2',
+        },
+        lokiSearch: {
+          datasourceUid: undefined,
+        },
+      },
+    });
+    const lokiDS4 = ds4.getLokiSearchDS();
+    expect(lokiDS4).toBe(undefined);
+  });
 });
 
 const backendSrvWithPrometheus = {

public/app/plugins/datasource/tempo/datasource.ts

@@ -45,6 +45,9 @@ export interface TempoJsonData extends DataSourceJsonData {
     hide?: boolean;
   };
   nodeGraph?: NodeGraphOptions;
+  lokiSearch?: {
+    datasourceUid?: string;
+  };
 }
 
 export interface TempoQuery extends DataQuery {
@@ -81,6 +84,9 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
     hide?: boolean;
   };
   nodeGraph?: NodeGraphOptions;
+  lokiSearch?: {
+    datasourceUid?: string;
+  };
   uploadedJson?: string | ArrayBuffer | null = null;
 
   constructor(private instanceSettings: DataSourceInstanceSettings<TempoJsonData>) {
@@ -89,6 +95,7 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
     this.serviceMap = instanceSettings.jsonData.serviceMap;
     this.search = instanceSettings.jsonData.search;
     this.nodeGraph = instanceSettings.jsonData.nodeGraph;
+    this.lokiSearch = instanceSettings.jsonData.lokiSearch;
   }
 
   query(options: DataQueryRequest<TempoQuery>): Observable<DataQueryResponse> {
@@ -100,11 +107,13 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
       return of({ data: [], state: LoadingState.Done });
     }
 
+    const logsDatasourceUid = this.getLokiSearchDS();
+
     // Run search queries on linked datasource
-    if (this.tracesToLogs?.datasourceUid && targets.search?.length > 0) {
+    if (logsDatasourceUid && targets.search?.length > 0) {
       const dsSrv = getDatasourceSrv();
       subQueries.push(
-        from(dsSrv.get(this.tracesToLogs.datasourceUid)).pipe(
+        from(dsSrv.get(logsDatasourceUid)).pipe(
           mergeMap((linkedDatasource: DataSourceApi) => {
             // Wrap linked query into a data request based on original request
             const linkedRequest: DataQueryRequest = { ...options, targets: targets.search.map((t) => t.linkedQuery!) };
@@ -300,6 +309,15 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
     const ds = await getDatasourceSrv().get(this.serviceMap!.datasourceUid);
     return ds.getTagValues!({ key });
   }
+
+  // Get linked loki search datasource. Fall back to legacy loki search/trace to logs config
+  getLokiSearchDS = (): string | undefined => {
+    const legacyLogsDatasourceUid =
+      this.tracesToLogs?.lokiSearch !== false && this.lokiSearch === undefined
+        ? this.tracesToLogs?.datasourceUid
+        : undefined;
+    return this.lokiSearch?.datasourceUid ?? legacyLogsDatasourceUid;
+  };
 }
 
 function queryServiceMapPrometheus(request: DataQueryRequest<PromQuery>, datasourceUid: string) {