Tempo: add ability to upload trace json (#37407)

* Tempo: upload json

* Add test for upload

* Minor changes

* Add docs

* Update docs/sources/datasources/tempo.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Add graphframes as well to upload

* Add comments to code

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>
This commit is contained in:
Zoltán Bedi
2021-08-05 15:13:44 +02:00
committed by GitHub
parent 5a54deb38b
commit e0010860bd
9 changed files with 690 additions and 11 deletions

View File

@@ -1,18 +1,29 @@
import { css } from '@emotion/css';
import { DataQuery, DataSourceApi, ExploreQueryFieldProps } from '@grafana/data';
import { selectors } from '@grafana/e2e-selectors';
import { getDataSourceSrv } from '@grafana/runtime';
import { InlineField, InlineFieldRow, InlineLabel, LegacyForms, RadioButtonGroup } from '@grafana/ui';
import {
FileDropzone,
InlineField,
InlineFieldRow,
InlineLabel,
LegacyForms,
RadioButtonGroup,
Themeable2,
withTheme2,
} from '@grafana/ui';
import { TraceToLogsOptions } from 'app/core/components/TraceToLogsSettings';
import React from 'react';
import { LokiQueryField } from '../loki/components/LokiQueryField';
import { TempoDatasource, TempoQuery, TempoQueryType } from './datasource';
type Props = ExploreQueryFieldProps<TempoDatasource, TempoQuery>;
interface Props extends ExploreQueryFieldProps<TempoDatasource, TempoQuery>, Themeable2 {}
const DEFAULT_QUERY_TYPE: TempoQueryType = 'traceId';
interface State {
linkedDatasource?: DataSourceApi;
}
export class TempoQueryField extends React.PureComponent<Props, State> {
class TempoQueryFieldComponent extends React.PureComponent<Props, State> {
state = {
linkedDatasource: undefined,
};
@@ -59,6 +70,7 @@ export class TempoQueryField extends React.PureComponent<Props, State> {
options={[
{ value: 'search', label: 'Search' },
{ value: 'traceId', label: 'TraceID' },
{ value: 'upload', label: 'JSON file' },
]}
value={query.queryType || DEFAULT_QUERY_TYPE}
onChange={(v) =>
@@ -89,7 +101,18 @@ export class TempoQueryField extends React.PureComponent<Props, State> {
{query.queryType === 'search' && !linkedDatasource && (
<div className="text-warning">Please set up a Traces-to-logs datasource in the datasource settings.</div>
)}
{query.queryType !== 'search' && (
{query.queryType === 'upload' && (
<div className={css({ padding: this.props.theme.spacing(2) })}>
<FileDropzone
options={{ multiple: false }}
onLoad={(result) => {
this.props.datasource.uploadedJson = result;
this.props.onRunQuery();
}}
/>
</div>
)}
{(!query.queryType || query.queryType === 'traceId') && (
<LegacyForms.FormField
label="Trace ID"
labelWidth={4}
@@ -117,3 +140,5 @@ export class TempoQueryField extends React.PureComponent<Props, State> {
);
}
}
export const TempoQueryField = withTheme2(TempoQueryFieldComponent);

View File

@@ -1,8 +1,16 @@
import { DataFrame, dataFrameToJSON, DataSourceInstanceSettings, MutableDataFrame, PluginType } from '@grafana/data';
import {
DataFrame,
dataFrameToJSON,
DataSourceInstanceSettings,
FieldType,
MutableDataFrame,
PluginType,
} from '@grafana/data';
import { BackendDataSourceResponse, FetchResponse, setBackendSrv } from '@grafana/runtime';
import { Observable, of } from 'rxjs';
import { createFetchResponse } from 'test/helpers/createFetchResponse';
import { TempoDatasource } from './datasource';
import { FetchResponse, setBackendSrv, BackendDataSourceResponse } from '@grafana/runtime';
import mockJson from './mockJsonResponse.json';
describe('Tempo data source', () => {
it('parses json fields from backend', async () => {
@@ -68,6 +76,21 @@ describe('Tempo data source', () => {
{ name: 'source', values: [] },
]);
});
it('should handle json file upload', async () => {
const ds = new TempoDatasource(defaultSettings);
ds.uploadedJson = JSON.stringify(mockJson);
const response = await ds
.query({
targets: [{ queryType: 'upload', refId: 'A' }],
} as any)
.toPromise();
const field = response.data[0].fields[0];
expect(field.name).toBe('traceID');
expect(field.type).toBe(FieldType.string);
expect(field.values.get(0)).toBe('60ba2abb44f13eae');
expect(field.values.length).toBe(6);
});
});
function setupBackendSrv(frame: DataFrame) {

View File

@@ -4,16 +4,17 @@ import {
DataQueryResponse,
DataSourceApi,
DataSourceInstanceSettings,
LoadingState,
} from '@grafana/data';
import { DataSourceWithBackend } from '@grafana/runtime';
import { TraceToLogsData, TraceToLogsOptions } from 'app/core/components/TraceToLogsSettings';
import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
import { from, merge, Observable, throwError } from 'rxjs';
import { from, merge, Observable, of, throwError } from 'rxjs';
import { map, mergeMap } from 'rxjs/operators';
import { LokiOptions } from '../loki/types';
import { transformTrace, transformTraceList } from './resultTransformer';
import { transformFromOTLP as transformFromOTEL, transformTrace, transformTraceList } from './resultTransformer';
export type TempoQueryType = 'search' | 'traceId';
export type TempoQueryType = 'search' | 'traceId' | 'upload';
export type TempoQuery = {
query: string;
@@ -24,6 +25,7 @@ export type TempoQuery = {
export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TraceToLogsData> {
tracesToLogs?: TraceToLogsOptions;
uploadedJson?: string | ArrayBuffer | null = null;
constructor(instanceSettings: DataSourceInstanceSettings<TraceToLogsData>) {
super(instanceSettings);
@@ -34,6 +36,7 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TraceToLo
const subQueries: Array<Observable<DataQueryResponse>> = [];
const filteredTargets = options.targets.filter((target) => !target.hide);
const searchTargets = filteredTargets.filter((target) => target.queryType === 'search');
const uploadTargets = filteredTargets.filter((target) => target.queryType === 'upload');
const traceTargets = filteredTargets.filter(
(target) => target.queryType === 'traceId' || target.queryType === undefined
);
@@ -68,6 +71,19 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TraceToLo
);
}
if (uploadTargets.length) {
if (this.uploadedJson) {
const otelTraceData = JSON.parse(this.uploadedJson as string);
if (!otelTraceData.batches) {
subQueries.push(of({ error: { message: 'JSON is not valid opentelemetry format' }, data: [] }));
} else {
subQueries.push(of(transformFromOTEL(otelTraceData.batches)));
}
} else {
subQueries.push(of({ data: [], state: LoadingState.Done }));
}
}
if (traceTargets.length > 0) {
const traceRequest: DataQueryRequest<TempoQuery> = { ...options, targets: traceTargets };
subQueries.push(

View File

@@ -0,0 +1,319 @@
{
"batches": [
{
"resource": {
"attributes": [
{ "key": "service.name", "value": { "stringValue": "db" } },
{ "key": "job", "value": { "stringValue": "tns/db" } },
{ "key": "opencensus.exporterversion", "value": { "stringValue": "Jaeger-Go-2.22.1" } },
{ "key": "host.name", "value": { "stringValue": "63d16772b4a2" } },
{ "key": "ip", "value": { "stringValue": "172.24.0.2" } },
{ "key": "client-uuid", "value": { "stringValue": "39fb01637a579639" } }
]
},
"instrumentationLibrarySpans": [
{
"instrumentationLibrary": {},
"spans": [
{
"traceId": "AAAAAAAAAABguiq7RPE+rg==",
"spanId": "cmteMBAvwNA=",
"parentSpanId": "OY8PIaPbma4=",
"name": "HTTP GET - root",
"kind": "SPAN_KIND_SERVER",
"startTimeUnixNano": "1627471657255809000",
"endTimeUnixNano": "1627471657256268000",
"attributes": [
{ "key": "http.status_code", "value": { "intValue": "200" } },
{ "key": "http.method", "value": { "stringValue": "GET" } },
{ "key": "http.url", "value": { "stringValue": "/" } },
{ "key": "component", "value": { "stringValue": "net/http" } }
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{ "key": "service.name", "value": { "stringValue": "app" } },
{ "key": "job", "value": { "stringValue": "tns/app" } },
{ "key": "opencensus.exporterversion", "value": { "stringValue": "Jaeger-Go-2.22.1" } },
{ "key": "host.name", "value": { "stringValue": "f68212e86151" } },
{ "key": "ip", "value": { "stringValue": "172.24.0.7" } },
{ "key": "client-uuid", "value": { "stringValue": "8b6e8600b53a24" } }
]
},
"instrumentationLibrarySpans": [
{
"instrumentationLibrary": {},
"spans": [
{
"traceId": "AAAAAAAAAABguiq7RPE+rg==",
"spanId": "GVdnHErYWoo=",
"parentSpanId": "WgdmnUQaoyY=",
"name": "HTTP Client",
"startTimeUnixNano": "1627471657251425000",
"endTimeUnixNano": "1627471657258789000",
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{ "key": "service.name", "value": { "stringValue": "lb" } },
{ "key": "job", "value": { "stringValue": "tns/loadgen" } },
{ "key": "opencensus.exporterversion", "value": { "stringValue": "Jaeger-Go-2.22.1" } },
{ "key": "host.name", "value": { "stringValue": "6475ba6d3c8b" } },
{ "key": "ip", "value": { "stringValue": "172.24.0.8" } },
{ "key": "client-uuid", "value": { "stringValue": "535eb863d7ade742" } }
]
},
"instrumentationLibrarySpans": [
{
"instrumentationLibrary": {},
"spans": [
{
"traceId": "AAAAAAAAAABguiq7RPE+rg==",
"spanId": "YLoqu0TxPq4=",
"name": "HTTP Client",
"startTimeUnixNano": "1627471657247632000",
"endTimeUnixNano": "1627471657260233000",
"attributes": [
{ "key": "sampler.type", "value": { "stringValue": "const" } },
{ "key": "sampler.param", "value": { "boolValue": true } }
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{ "key": "service.name", "value": { "stringValue": "lb" } },
{ "key": "job", "value": { "stringValue": "tns/loadgen" } },
{ "key": "opencensus.exporterversion", "value": { "stringValue": "Jaeger-Go-2.22.1" } },
{ "key": "host.name", "value": { "stringValue": "6475ba6d3c8b" } },
{ "key": "ip", "value": { "stringValue": "172.24.0.8" } },
{ "key": "client-uuid", "value": { "stringValue": "535eb863d7ade742" } }
]
},
"instrumentationLibrarySpans": [
{
"instrumentationLibrary": {},
"spans": [
{
"traceId": "AAAAAAAAAABguiq7RPE+rg==",
"spanId": "VzhhbsaOedI=",
"parentSpanId": "YLoqu0TxPq4=",
"name": "HTTP GET",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1627471657247674000",
"endTimeUnixNano": "1627471657261178000",
"attributes": [
{ "key": "http.status_code", "value": { "intValue": "200" } },
{ "key": "component", "value": { "stringValue": "net/http" } },
{ "key": "http.method", "value": { "stringValue": "GET" } },
{ "key": "http.url", "value": { "stringValue": "app:80" } },
{ "key": "net/http.reused", "value": { "boolValue": false } },
{ "key": "net/http.was_idle", "value": { "boolValue": false } }
],
"events": [
{
"timeUnixNano": "1627471657247711000",
"attributes": [{ "key": "event", "value": { "stringValue": "GetConn" } }]
},
{
"timeUnixNano": "1627471657247822000",
"attributes": [
{ "key": "event", "value": { "stringValue": "DNSStart" } },
{ "key": "host", "value": { "stringValue": "app" } }
]
},
{
"timeUnixNano": "1627471657249309000",
"attributes": [
{ "key": "event", "value": { "stringValue": "DNSDone" } },
{ "key": "addr", "value": { "stringValue": "172.24.0.7" } }
]
},
{
"timeUnixNano": "1627471657249395000",
"attributes": [
{ "key": "event", "value": { "stringValue": "ConnectStart" } },
{ "key": "network", "value": { "stringValue": "tcp" } },
{ "key": "addr", "value": { "stringValue": "172.24.0.7:80" } }
]
},
{
"timeUnixNano": "1627471657250250000",
"attributes": [
{ "key": "event", "value": { "stringValue": "ConnectDone" } },
{ "key": "network", "value": { "stringValue": "tcp" } },
{ "key": "addr", "value": { "stringValue": "172.24.0.7:80" } }
]
},
{
"timeUnixNano": "1627471657250313000",
"attributes": [{ "key": "event", "value": { "stringValue": "GotConn" } }]
},
{
"timeUnixNano": "1627471657250446000",
"attributes": [{ "key": "event", "value": { "stringValue": "WroteHeaders" } }]
},
{
"timeUnixNano": "1627471657250465000",
"attributes": [{ "key": "event", "value": { "stringValue": "WroteRequest" } }]
},
{
"timeUnixNano": "1627471657260087000",
"attributes": [{ "key": "event", "value": { "stringValue": "GotFirstResponseByte" } }]
},
{
"timeUnixNano": "1627471657260804000",
"attributes": [{ "key": "event", "value": { "stringValue": "ClosedBody" } }]
}
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{ "key": "service.name", "value": { "stringValue": "app" } },
{ "key": "job", "value": { "stringValue": "tns/app" } },
{ "key": "opencensus.exporterversion", "value": { "stringValue": "Jaeger-Go-2.22.1" } },
{ "key": "host.name", "value": { "stringValue": "f68212e86151" } },
{ "key": "ip", "value": { "stringValue": "172.24.0.7" } },
{ "key": "client-uuid", "value": { "stringValue": "8b6e8600b53a24" } }
]
},
"instrumentationLibrarySpans": [
{
"instrumentationLibrary": {},
"spans": [
{
"traceId": "AAAAAAAAAABguiq7RPE+rg==",
"spanId": "OY8PIaPbma4=",
"parentSpanId": "GVdnHErYWoo=",
"name": "HTTP GET",
"kind": "SPAN_KIND_CLIENT",
"startTimeUnixNano": "1627471657251445000",
"endTimeUnixNano": "1627471657260828000",
"attributes": [
{ "key": "http.status_code", "value": { "intValue": "200" } },
{ "key": "component", "value": { "stringValue": "net/http" } },
{ "key": "http.method", "value": { "stringValue": "GET" } },
{ "key": "http.url", "value": { "stringValue": "db:80" } },
{ "key": "net/http.reused", "value": { "boolValue": false } },
{ "key": "net/http.was_idle", "value": { "boolValue": false } }
],
"events": [
{
"timeUnixNano": "1627471657251575000",
"attributes": [{ "key": "event", "value": { "stringValue": "GetConn" } }]
},
{
"timeUnixNano": "1627471657251700000",
"attributes": [
{ "key": "event", "value": { "stringValue": "DNSStart" } },
{ "key": "host", "value": { "stringValue": "db" } }
]
},
{
"timeUnixNano": "1627471657254144000",
"attributes": [
{ "key": "event", "value": { "stringValue": "DNSDone" } },
{ "key": "addr", "value": { "stringValue": "172.24.0.2" } }
]
},
{
"timeUnixNano": "1627471657254295000",
"attributes": [
{ "key": "event", "value": { "stringValue": "ConnectStart" } },
{ "key": "network", "value": { "stringValue": "tcp" } },
{ "key": "addr", "value": { "stringValue": "172.24.0.2:80" } }
]
},
{
"timeUnixNano": "1627471657255054000",
"attributes": [
{ "key": "event", "value": { "stringValue": "ConnectDone" } },
{ "key": "network", "value": { "stringValue": "tcp" } },
{ "key": "addr", "value": { "stringValue": "172.24.0.2:80" } }
]
},
{
"timeUnixNano": "1627471657255199000",
"attributes": [{ "key": "event", "value": { "stringValue": "GotConn" } }]
},
{
"timeUnixNano": "1627471657255257000",
"attributes": [{ "key": "event", "value": { "stringValue": "WroteHeaders" } }]
},
{
"timeUnixNano": "1627471657255274000",
"attributes": [{ "key": "event", "value": { "stringValue": "WroteRequest" } }]
},
{
"timeUnixNano": "1627471657258308000",
"attributes": [{ "key": "event", "value": { "stringValue": "GotFirstResponseByte" } }]
},
{
"timeUnixNano": "1627471657260811000",
"attributes": [{ "key": "event", "value": { "stringValue": "ClosedBody" } }]
}
],
"status": {}
}
]
}
]
},
{
"resource": {
"attributes": [
{ "key": "service.name", "value": { "stringValue": "app" } },
{ "key": "job", "value": { "stringValue": "tns/app" } },
{ "key": "opencensus.exporterversion", "value": { "stringValue": "Jaeger-Go-2.22.1" } },
{ "key": "host.name", "value": { "stringValue": "f68212e86151" } },
{ "key": "ip", "value": { "stringValue": "172.24.0.7" } },
{ "key": "client-uuid", "value": { "stringValue": "8b6e8600b53a24" } }
]
},
"instrumentationLibrarySpans": [
{
"instrumentationLibrary": {},
"spans": [
{
"traceId": "AAAAAAAAAABguiq7RPE+rg==",
"spanId": "WgdmnUQaoyY=",
"parentSpanId": "VzhhbsaOedI=",
"name": "HTTP GET - root",
"kind": "SPAN_KIND_SERVER",
"startTimeUnixNano": "1627471657251228000",
"endTimeUnixNano": "1627471657260930000",
"attributes": [
{ "key": "http.status_code", "value": { "intValue": "200" } },
{ "key": "http.method", "value": { "stringValue": "GET" } },
{ "key": "http.url", "value": { "stringValue": "/" } },
{ "key": "component", "value": { "stringValue": "net/http" } }
],
"status": {}
}
]
}
]
}
]
}

View File

@@ -1,4 +1,17 @@
import { DataQueryResponse, ArrayVector, DataFrame, Field, FieldType, MutableDataFrame } from '@grafana/data';
import {
ArrayVector,
DataFrame,
DataQueryResponse,
Field,
FieldType,
MutableDataFrame,
TraceKeyValuePair,
TraceLog,
TraceSpanRow,
} from '@grafana/data';
import { SpanKind, SpanStatusCode } from '@opentelemetry/api';
import { collectorTypes } from '@opentelemetry/exporter-collector';
import { ResourceAttributes } from '@opentelemetry/semantic-conventions';
import { createGraphFrames } from './graphTransform';
export function createTableFrame(
@@ -98,6 +111,168 @@ export function transformTraceList(
return response;
}
// Don't forget to change the backend code when the id representation changed
function transformBase64IDToHexString(base64: string) {
const buffer = Buffer.from(base64, 'base64');
const id = buffer.toString('hex');
return id.length > 16 ? id.slice(16) : id;
}
function getAttributeValue(value: collectorTypes.opentelemetryProto.common.v1.AnyValue): any {
if (value.stringValue) {
return value.stringValue;
}
if (value.boolValue !== undefined) {
return Boolean(value.boolValue);
}
if (value.intValue !== undefined) {
return Number.parseInt(value.intValue as any, 10);
}
if (value.doubleValue) {
return Number.parseFloat(value.doubleValue as any);
}
if (value.arrayValue) {
const arrayValue = [];
for (const arValue of value.arrayValue.values) {
arrayValue.push(getAttributeValue(arValue));
}
return arrayValue;
}
return '';
}
function resourceToProcess(resource: collectorTypes.opentelemetryProto.resource.v1.Resource | undefined) {
const serviceTags: TraceKeyValuePair[] = [];
let serviceName = 'OTLPResourceNoServiceName';
if (!resource) {
return { serviceName, serviceTags };
}
for (const attribute of resource.attributes) {
if (attribute.key === ResourceAttributes.SERVICE_NAME) {
serviceName = attribute.value.stringValue || serviceName;
}
serviceTags.push({ key: attribute.key, value: getAttributeValue(attribute.value) });
}
return { serviceName, serviceTags };
}
function getSpanTags(
span: collectorTypes.opentelemetryProto.trace.v1.Span,
instrumentationLibrary?: collectorTypes.opentelemetryProto.common.v1.InstrumentationLibrary
): TraceKeyValuePair[] {
const spanTags: TraceKeyValuePair[] = [];
if (instrumentationLibrary) {
if (instrumentationLibrary.name) {
spanTags.push({ key: 'otel.library.name', value: instrumentationLibrary.name });
}
if (instrumentationLibrary.version) {
spanTags.push({ key: 'otel.library.version', value: instrumentationLibrary.version });
}
}
if (span.attributes) {
for (const attribute of span.attributes) {
spanTags.push({ key: attribute.key, value: getAttributeValue(attribute.value) });
}
}
if (span.status) {
if (span.status.code && (span.status.code as any) !== SpanStatusCode.UNSET) {
spanTags.push({
key: 'otel.status_code',
value: SpanStatusCode[span.status.code],
});
if (span.status.message) {
spanTags.push({ key: 'otel.status_description', value: span.status.message });
}
}
if (span.status.code === SpanStatusCode.ERROR) {
spanTags.push({ key: 'error', value: true });
}
}
if (
span.kind !== undefined &&
span.kind !== collectorTypes.opentelemetryProto.trace.v1.Span.SpanKind.SPAN_KIND_INTERNAL
) {
spanTags.push({
key: 'span.kind',
value: SpanKind[collectorTypes.opentelemetryProto.trace.v1.Span.SpanKind[span.kind] as any].toLowerCase(),
});
}
return spanTags;
}
function getLogs(span: collectorTypes.opentelemetryProto.trace.v1.Span) {
const logs: TraceLog[] = [];
if (span.events) {
for (const event of span.events) {
const fields: TraceKeyValuePair[] = [];
if (event.attributes) {
for (const attribute of event.attributes) {
fields.push({ key: attribute.key, value: getAttributeValue(attribute.value) });
}
}
logs.push({ fields, timestamp: event.timeUnixNano / 1000000 });
}
}
return logs;
}
export function transformFromOTLP(
traceData: collectorTypes.opentelemetryProto.trace.v1.ResourceSpans[]
): DataQueryResponse {
const frame = new MutableDataFrame({
fields: [
{ name: 'traceID', type: FieldType.string },
{ name: 'spanID', type: FieldType.string },
{ name: 'parentSpanID', type: FieldType.string },
{ name: 'operationName', type: FieldType.string },
{ name: 'serviceName', type: FieldType.string },
{ name: 'serviceTags', type: FieldType.other },
{ name: 'startTime', type: FieldType.number },
{ name: 'duration', type: FieldType.number },
{ name: 'logs', type: FieldType.other },
{ name: 'tags', type: FieldType.other },
],
meta: {
preferredVisualisationType: 'trace',
},
});
for (const data of traceData) {
const { serviceName, serviceTags } = resourceToProcess(data.resource);
for (const librarySpan of data.instrumentationLibrarySpans) {
for (const span of librarySpan.spans) {
frame.add({
traceID: transformBase64IDToHexString(span.traceId),
spanID: transformBase64IDToHexString(span.spanId),
parentSpanID: transformBase64IDToHexString(span.parentSpanId || ''),
operationName: span.name || '',
serviceName,
serviceTags,
startTime: span.startTimeUnixNano! / 1000000,
duration: (span.endTimeUnixNano! - span.startTimeUnixNano!) / 1000000,
tags: getSpanTags(span, librarySpan.instrumentationLibrary),
logs: getLogs(span),
} as TraceSpanRow);
}
}
}
return { data: [frame, ...createGraphFrames(frame)] };
}
export function transformTrace(response: DataQueryResponse): DataQueryResponse {
// We need to parse some of the fields which contain stringified json.
// Seems like we can't just map the values as the frame we got from backend has some default processing