Trace UI demo (#20297)

* Add integration with Jeager
Add Jaeger datasource and modify derived fields in loki to allow for opening a trace in Jager in separate split.
Modifies build so that this branch docker images are pushed to docker hub
Add a traceui dir with docker-compose and provision files for demoing.:wq

* Enable docker logger plugin to send logs to loki

* Add placeholder zipkin datasource

* Fixed rebase issues, added enhanceDataFrame to non-legacy code path

* Trace selector for jaeger query field

* Fix logs default mode for Loki

* Fix loading jaeger query field services on split

* Updated grafana image in traceui/compose file

* Fix prettier error

* Hide behind feature flag, clean up unused code.

* Fix tests

* Fix tests

* Cleanup code and review feedback

* Remove traceui directory

* Remove circle build changes

* Fix feature toggles object

* Fix merge issues

* Fix some null errors

* Fix test after strict null changes

* Review feedback fixes

* Fix toggle name

Co-authored-by: David Kaltschmidt <david.kaltschmidt@gmail.com>

public/app/plugins/datasource/jaeger/ConfigEditor.tsx

@@ -0,0 +1,18 @@
+import React from 'react';
+import { DataSourcePluginOptionsEditorProps } from '@grafana/data';
+import { DataSourceHttpSettings } from '@grafana/ui';
+
+export type Props = DataSourcePluginOptionsEditorProps;
+
+export const ConfigEditor: React.FC<Props> = ({ options, onOptionsChange }) => {
+  return (
+    <>
+      <DataSourceHttpSettings
+        defaultUrl={'http://localhost:16686'}
+        dataSourceConfig={options}
+        showAccessOptions={true}
+        onChange={onOptionsChange}
+      />
+    </>
+  );
+};

public/app/plugins/datasource/jaeger/QueryField.tsx

@@ -0,0 +1,202 @@
+import React from 'react';
+import { JaegerDatasource, JaegerQuery } from './datasource';
+import { ButtonCascader, CascaderOption } from '@grafana/ui';
+
+import { ExploreQueryFieldProps } from '@grafana/data';
+
+const ALL_OPERATIONS_KEY = '__ALL__';
+const NO_TRACES_KEY = '__NO_TRACES__';
+
+type Props = ExploreQueryFieldProps<JaegerDatasource, JaegerQuery>;
+interface State {
+  serviceOptions: CascaderOption[];
+}
+
+function getLabelFromTrace(trace: any): string {
+  const firstSpan = trace.spans && trace.spans[0];
+  if (firstSpan) {
+    return `${firstSpan.operationName} [${firstSpan.duration} ms]`;
+  }
+  return trace.traceID;
+}
+
+export class JaegerQueryField extends React.PureComponent<Props, State> {
+  constructor(props: Props, context: React.Context<any>) {
+    super(props, context);
+    this.state = {
+      serviceOptions: [],
+    };
+  }
+
+  componentDidMount() {
+    this.getServices();
+  }
+
+  async getServices() {
+    const url = '/api/services';
+    const { datasource } = this.props;
+    try {
+      const res = await datasource.metadataRequest(url);
+      if (res) {
+        const services = res as string[];
+        const serviceOptions: CascaderOption[] = services.sort().map(service => ({
+          label: service,
+          value: service,
+          isLeaf: false,
+        }));
+        this.setState({ serviceOptions });
+      }
+    } catch (error) {
+      console.error(error);
+    }
+  }
+
+  onLoadOptions = async (selectedOptions: CascaderOption[]) => {
+    const service = selectedOptions[0].value;
+    if (selectedOptions.length === 1) {
+      // Load operations
+      const operations: string[] = await this.findOperations(service);
+      const allOperationsOption: CascaderOption = {
+        label: '[ALL]',
+        value: ALL_OPERATIONS_KEY,
+      };
+      const operationOptions: CascaderOption[] = [
+        allOperationsOption,
+        ...operations.sort().map(operation => ({
+          label: operation,
+          value: operation,
+          isLeaf: false,
+        })),
+      ];
+      this.setState(state => {
+        const serviceOptions = state.serviceOptions.map(serviceOption => {
+          if (serviceOption.value === service) {
+            return {
+              ...serviceOption,
+              children: operationOptions,
+            };
+          }
+          return serviceOption;
+        });
+        return { serviceOptions };
+      });
+    } else if (selectedOptions.length === 2) {
+      // Load traces
+      const operationValue = selectedOptions[1].value;
+      const operation = operationValue === ALL_OPERATIONS_KEY ? '' : operationValue;
+      const traces: any[] = await this.findTraces(service, operation);
+      let traceOptions: CascaderOption[] = traces.map(trace => ({
+        label: getLabelFromTrace(trace),
+        value: trace.traceID,
+      }));
+      if (traceOptions.length === 0) {
+        traceOptions = [
+          {
+            label: '[No traces in time range]',
+            value: NO_TRACES_KEY,
+          },
+        ];
+      }
+      this.setState(state => {
+        // Place new traces into the correct service/operation sub-tree
+        const serviceOptions = state.serviceOptions.map(serviceOption => {
+          if (serviceOption.value === service) {
+            const operationOptions = serviceOption.children.map(operationOption => {
+              if (operationOption.value === operationValue) {
+                return {
+                  ...operationOption,
+                  children: traceOptions,
+                };
+              }
+              return operationOption;
+            });
+            return {
+              ...serviceOption,
+              children: operationOptions,
+            };
+          }
+          return serviceOption;
+        });
+        return { serviceOptions };
+      });
+    }
+  };
+
+  findOperations = async (service: string) => {
+    const { datasource } = this.props;
+    const url = `/api/services/${service}/operations`;
+    try {
+      return await datasource.metadataRequest(url);
+    } catch (error) {
+      console.error(error);
+    }
+    return [];
+  };
+
+  findTraces = async (service: string, operation?: string) => {
+    const { datasource } = this.props;
+    const { start, end } = datasource.getTimeRange();
+
+    const traceSearch = {
+      start,
+      end,
+      service,
+      operation,
+      limit: 10,
+      lookback: '1h',
+      maxDuration: '',
+      minDuration: '',
+    };
+    const url = '/api/traces';
+    try {
+      return await datasource.metadataRequest(url, traceSearch);
+    } catch (error) {
+      console.error(error);
+    }
+    return [];
+  };
+
+  onSelectTrace = (values: string[], selectedOptions: CascaderOption[]) => {
+    const { query, onChange, onRunQuery } = this.props;
+    if (selectedOptions.length === 3) {
+      const traceID = selectedOptions[2].value;
+      onChange({ ...query, query: traceID });
+      onRunQuery();
+    }
+  };
+
+  render() {
+    const { query, onChange } = this.props;
+    const { serviceOptions } = this.state;
+
+    return (
+      <>
+        <div className="gf-form-inline gf-form-inline--nowrap">
+          <div className="gf-form flex-shrink-0">
+            <ButtonCascader options={serviceOptions} onChange={this.onSelectTrace} loadData={this.onLoadOptions}>
+              Traces
+            </ButtonCascader>
+          </div>
+          <div className="gf-form gf-form--grow flex-shrink-1">
+            <div className={'slate-query-field__wrapper'}>
+              <div className="slate-query-field">
+                <input
+                  style={{ width: '100%' }}
+                  value={query.query || ''}
+                  onChange={e =>
+                    onChange({
+                      ...query,
+                      query: e.currentTarget.value,
+                    })
+                  }
+                />
+              </div>
+            </div>
+          </div>
+        </div>
+      </>
+    );
+  }
+}
+
+export default JaegerQueryField;

public/app/plugins/datasource/jaeger/datasource.ts

@@ -0,0 +1,85 @@
+import {
+  dateMath,
+  DateTime,
+  MutableDataFrame,
+  DataSourceApi,
+  DataSourceInstanceSettings,
+  DataQueryRequest,
+  DataQueryResponse,
+  DataQuery,
+} from '@grafana/data';
+import { getBackendSrv } from '@grafana/runtime';
+
+import { getTimeSrv } from 'app/features/dashboard/services/TimeSrv';
+import { DatasourceRequestOptions } from 'app/core/services/backend_srv';
+import { serializeParams } from '../../../core/utils/fetch';
+
+import { Observable, from, of } from 'rxjs';
+
+export type JaegerQuery = {
+  query: string;
+} & DataQuery;
+
+export class JaegerDatasource extends DataSourceApi<JaegerQuery> {
+  constructor(private instanceSettings: DataSourceInstanceSettings) {
+    super(instanceSettings);
+  }
+
+  _request(apiUrl: string, data?: any, options?: DatasourceRequestOptions): Observable<Record<string, any>> {
+    // Hack for proxying metadata requests
+    const baseUrl = `/api/datasources/proxy/${this.instanceSettings.id}`;
+    const params = data ? serializeParams(data) : '';
+    const url = `${baseUrl}${apiUrl}${params.length ? `?${params}` : ''}`;
+    const req = {
+      ...options,
+      url,
+    };
+
+    return from(getBackendSrv().datasourceRequest(req));
+  }
+
+  async metadataRequest(url: string, params?: Record<string, any>) {
+    const res = await this._request(url, params, { silent: true }).toPromise();
+    return res.data.data;
+  }
+
+  query(options: DataQueryRequest<JaegerQuery>): Observable<DataQueryResponse> {
+    //http://localhost:16686/search?end=1573338717880000&limit=20&lookback=6h&maxDuration&minDuration&service=app&start=1573317117880000
+    const url =
+      options.targets.length && options.targets[0].query
+        ? `${this.instanceSettings.url}/trace/${options.targets[0].query}?uiEmbed=v0`
+        : '';
+
+    return of({
+      data: [
+        new MutableDataFrame({
+          fields: [
+            {
+              name: 'url',
+              values: [url],
+            },
+          ],
+        }),
+      ],
+    });
+  }
+
+  async testDatasource(): Promise<any> {
+    return true;
+  }
+
+  getTime(date: string | DateTime, roundUp: boolean) {
+    if (typeof date === 'string') {
+      date = dateMath.parse(date, roundUp);
+    }
+    return date.valueOf() * 1000;
+  }
+
+  getTimeRange(): { start: number; end: number } {
+    const range = getTimeSrv().timeRange();
+    return {
+      start: this.getTime(range.from, false),
+      end: this.getTime(range.to, true),
+    };
+  }
+}

public/app/plugins/datasource/jaeger/module.ts

@@ -0,0 +1,8 @@
+import { DataSourcePlugin } from '@grafana/data';
+import { JaegerDatasource } from './datasource';
+import { JaegerQueryField } from './QueryField';
+import { ConfigEditor } from './ConfigEditor';
+
+export const plugin = new DataSourcePlugin(JaegerDatasource)
+  .setConfigEditor(ConfigEditor)
+  .setExploreQueryField(JaegerQueryField);

public/app/plugins/datasource/jaeger/plugin.json

@@ -0,0 +1,35 @@
+{
+  "type": "datasource",
+  "name": "Jaeger",
+  "id": "jaeger",
+  "category": "tracing",
+
+  "metrics": false,
+  "alerting": false,
+  "annotations": false,
+  "logs": false,
+  "streaming": false,
+  "tracing": true,
+
+  "info": {
+    "description": "Open source, end-to-end distributed tracing",
+    "author": {
+      "name": "Grafana Project",
+      "url": "https://grafana.com"
+    },
+    "logos": {
+      "small": "img/jaeger_logo.svg",
+      "large": "img/jaeger_logo.svg"
+    },
+    "links": [
+      {
+        "name": "Learn more",
+        "url": "https://www.jaegertracing.io"
+      },
+      {
+        "name": "GitHub Project",
+        "url": "https://github.com/jaegertracing/jaeger"
+      }
+    ]
+  }
+}

public/app/plugins/datasource/loki/configuration/DebugSection.tsx

@@ -4,7 +4,7 @@ import cx from 'classnames';
 import { FormField } from '@grafana/ui';
 import { DerivedFieldConfig } from '../types';
 import { getLinksFromLogsField } from '../../../../features/panel/panellinks/linkSuppliers';
-import { ArrayVector, FieldType } from '@grafana/data';
+import { ArrayVector, Field, FieldType, LinkModel } from '@grafana/data';
 
 type Props = {
   derivedFields: DerivedFieldConfig[];
@@ -90,7 +90,7 @@ function makeDebugFields(derivedFields: DerivedFieldConfig[], debugText: string)
       try {
         const testMatch = debugText.match(field.matcherRegex);
         const value = testMatch && testMatch[1];
-        let link;
+        let link: LinkModel<Field>;
 
         if (field.url && value) {
           link = getLinksFromLogsField(
@@ -103,7 +103,7 @@ function makeDebugFields(derivedFields: DerivedFieldConfig[], debugText: string)
               },
             },
             0
-          )[0];
+          )[0].linkModel;
         }
 
         return {

public/app/plugins/datasource/loki/configuration/DerivedField.tsx

@@ -1,11 +1,16 @@
-import React from 'react';
+import React, { useState } from 'react';
 import { css } from 'emotion';
-import { Button, FormField, DataLinkInput, stylesFactory } from '@grafana/ui';
+import { Button, FormField, DataLinkInput, stylesFactory, Switch } from '@grafana/ui';
 import { VariableSuggestion } from '@grafana/data';
+import { DataSourceSelectItem } from '@grafana/data';
+
 import { DerivedFieldConfig } from '../types';
+import DataSourcePicker from 'app/core/components/Select/DataSourcePicker';
+import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
+import { config } from 'app/core/config';
 
 const getStyles = stylesFactory(() => ({
-  firstRow: css`
+  row: css`
     display: flex;
     align-items: baseline;
   `,
@@ -27,6 +32,7 @@ type Props = {
 export const DerivedField = (props: Props) => {
   const { value, onChange, onDelete, suggestions, className } = props;
   const styles = getStyles();
+  const [hasIntenalLink, setHasInternalLink] = useState(!!value.datasourceName);
 
   const handleChange = (field: keyof typeof value) => (event: React.ChangeEvent<HTMLInputElement>) => {
     onChange({
@@ -37,7 +43,7 @@ export const DerivedField = (props: Props) => {
 
   return (
     <div className={className}>
-      <div className={styles.firstRow}>
+      <div className={styles.row}>
         <FormField
           className={styles.nameField}
           labelWidth={5}
@@ -93,6 +99,64 @@ export const DerivedField = (props: Props) => {
           width: 100%;
         `}
       />
+
+      {config.featureToggles.tracingIntegration && (
+        <div className={styles.row}>
+          <Switch
+            label="Internal link"
+            checked={hasIntenalLink}
+            onChange={() => {
+              if (hasIntenalLink) {
+                onChange({
+                  ...value,
+                  datasourceName: undefined,
+                });
+              }
+              setHasInternalLink(!hasIntenalLink);
+            }}
+          />
+
+          {hasIntenalLink && (
+            <DataSourceSection
+              onChange={datasourceName => {
+                onChange({
+                  ...value,
+                  datasourceName,
+                });
+              }}
+              datasourceName={value.datasourceName}
+            />
+          )}
+        </div>
+      )}
     </div>
   );
 };
+
+type DataSourceSectionProps = {
+  datasourceName?: string;
+  onChange: (name: string) => void;
+};
+const DataSourceSection = (props: DataSourceSectionProps) => {
+  const { datasourceName, onChange } = props;
+  const datasources: DataSourceSelectItem[] = getDatasourceSrv()
+    .getExternal()
+    .map(
+      (ds: any) =>
+        ({
+          value: ds.name,
+          name: ds.name,
+          meta: ds.meta,
+        } as DataSourceSelectItem)
+    );
+  const selectedDatasource = datasourceName && datasources.find(d => d.name === datasourceName);
+  return (
+    <DataSourcePicker
+      onChange={newValue => {
+        onChange(newValue.name);
+      }}
+      datasources={datasources}
+      current={selectedDatasource}
+    />
+  );
+};

public/app/plugins/datasource/loki/datasource.ts

@@ -51,6 +51,7 @@ import {
 } from './types';
 import { LegacyTarget, LiveStreams } from './live_streams';
 import LanguageProvider from './language_provider';
+import { serializeParams } from '../../../core/utils/fetch';
 
 export type RangeQueryOptions = Pick<DataQueryRequest<LokiQuery>, 'range' | 'intervalMs' | 'maxDataPoints' | 'reverse'>;
 export const DEFAULT_MAX_LINES = 1000;
@@ -68,12 +69,6 @@ const DEFAULT_QUERY_PARAMS: Partial<LokiLegacyQueryRequest> = {
   query: '',
 };
 
-function serializeParams(data: Record<string, any>) {
-  return Object.keys(data)
-    .map(k => `${encodeURIComponent(k)}=${encodeURIComponent(data[k])}`)
-    .join('&');
-}
-
 interface LokiContextQueryOptions {
   direction?: 'BACKWARD' | 'FORWARD';
   limit?: number;

public/app/plugins/datasource/loki/result_transformer.ts

@@ -395,11 +395,16 @@ export const enhanceDataFrame = (dataFrame: DataFrame, config: LokiOptions | nul
 
   const fields = derivedFields.reduce((acc, field) => {
     const config: FieldConfig = {};
-    if (field.url) {
+    if (field.url || field.datasourceName) {
       config.links = [
         {
           url: field.url,
           title: '',
+          meta: field.datasourceName
+            ? {
+                datasourceName: field.datasourceName,
+              }
+            : undefined,
         },
       ];
     }

public/app/plugins/datasource/loki/types.ts

@@ -127,6 +127,7 @@ export type DerivedFieldConfig = {
   matcherRegex: string;
   name: string;
   url?: string;
+  datasourceName?: string;
 };
 
 export interface TransformerOptions {

public/app/plugins/datasource/zipkin/ConfigEditor.tsx

@@ -0,0 +1,16 @@
+import React from 'react';
+import { DataSourcePluginOptionsEditorProps } from '@grafana/data';
+import { DataSourceHttpSettings } from '@grafana/ui';
+
+export type Props = DataSourcePluginOptionsEditorProps;
+
+export const ConfigEditor: React.FC<Props> = ({ options, onOptionsChange }) => {
+  return (
+    <DataSourceHttpSettings
+      defaultUrl={'http://localhost:3100'}
+      dataSourceConfig={options}
+      showAccessOptions={true}
+      onChange={onOptionsChange}
+    />
+  );
+};

public/app/plugins/datasource/zipkin/QueryField.tsx

@@ -0,0 +1,22 @@
+import React from 'react';
+import { ZipkinDatasource, ZipkinQuery } from './datasource';
+import { ExploreQueryFieldProps } from '@grafana/data';
+
+type Props = ExploreQueryFieldProps<ZipkinDatasource, ZipkinQuery>;
+
+export const QueryField = (props: Props) => (
+  <div className={'slate-query-field__wrapper'}>
+    <div className="slate-query-field">
+      <input
+        style={{ width: '100%' }}
+        value={props.query.query || ''}
+        onChange={e =>
+          props.onChange({
+            ...props.query,
+            query: e.currentTarget.value,
+          })
+        }
+      />
+    </div>
+  </div>
+);

public/app/plugins/datasource/zipkin/datasource.ts

@@ -0,0 +1,38 @@
+import {
+  MutableDataFrame,
+  DataSourceApi,
+  DataSourceInstanceSettings,
+  DataQueryRequest,
+  DataQueryResponse,
+  DataQuery,
+} from '@grafana/data';
+import { Observable, of } from 'rxjs';
+
+export type ZipkinQuery = {
+  query: string;
+} & DataQuery;
+
+export class ZipkinDatasource extends DataSourceApi<ZipkinQuery> {
+  constructor(instanceSettings: DataSourceInstanceSettings) {
+    super(instanceSettings);
+  }
+
+  query(options: DataQueryRequest<ZipkinQuery>): Observable<DataQueryResponse> {
+    return of({
+      data: [
+        new MutableDataFrame({
+          fields: [
+            {
+              name: 'url',
+              values: [],
+            },
+          ],
+        }),
+      ],
+    });
+  }
+
+  async testDatasource(): Promise<any> {
+    return true;
+  }
+}

public/app/plugins/datasource/zipkin/module.ts

@@ -0,0 +1,8 @@
+import { DataSourcePlugin } from '@grafana/data';
+import { ZipkinDatasource } from './datasource';
+import { QueryField } from './QueryField';
+import { ConfigEditor } from './ConfigEditor';
+
+export const plugin = new DataSourcePlugin(ZipkinDatasource)
+  .setConfigEditor(ConfigEditor)
+  .setExploreQueryField(QueryField);

public/app/plugins/datasource/zipkin/plugin.json

@@ -0,0 +1,31 @@
+{
+  "type": "datasource",
+  "name": "Zipkin",
+  "id": "zipkin",
+  "category": "tracing",
+
+  "metrics": false,
+  "alerting": false,
+  "annotations": false,
+  "logs": false,
+  "streaming": false,
+  "tracing": true,
+
+  "info": {
+    "description": "Placeholder for the distributed tracing system.",
+    "author": {
+      "name": "Grafana Project",
+      "url": "https://grafana.com"
+    },
+    "logos": {
+      "small": "img/zipkin-logo.svg",
+      "large": "img/zipkin-logo.svg"
+    },
+    "links": [
+      {
+        "name": "Learn more",
+        "url": "https://zipkin.io"
+      }
+    ]
+  }
+}