Tempo: Traces to Logs - From viewing Tempo traces to displaying Splunk logs (#46855)

* Add functionality to see splunk logs via tempo trace view
This commit is contained in:
Cat Perry 2022-04-04 13:24:33 -07:00 committed by GitHub
parent 33006436cc
commit b5f2f3e710
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 281 additions and 71 deletions

View File

@ -28,15 +28,15 @@ To access Jaeger settings, click the **Configuration** (gear) icon, then click *
> **Note:** This feature is available in Grafana 7.4+.
This is a configuration for the [trace to logs feature]({{< relref "../explore/trace-integration" >}}). Select target data source (at this moment limited to Loki data sources) and select which tags will be used in the logs query.
This is a configuration for the [trace to logs feature]({{< relref "../explore/trace-integration" >}}). Select target data source (at this moment limited to Loki and Splunk \[logs\] data sources) and select which tags will be used in the logs query.
- **Data source -** Target data source.
- **Tags -** The tags that will be used in the Loki query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
- **Map tag names -** When enabled, allows configuring how Jaeger tag names map to Loki label names. For example, map `service.name` to `service`.
- **Span start time shift -** Shift in the start time for the Loki query based on the span start time. In order to extend to the past, you need to use a negative value. Use time interval units like 5s, 1m, 3h. The default is 0.
- **Span end time shift -** Shift in the end time for the Loki query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
- **Filter by Trace ID -** Toggle to append the trace ID to the Loki query.
- **Filter by Span ID -** Toggle to append the span ID to the Loki query.
- **Tags -** The tags that will be used in the logs query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
- **Map tag names -** When enabled, allows configuring how Jaeger tag names map to logs label names. For example, map `service.name` to `service`.
- **Span start time shift -** Shift in the start time for the logs query based on the span start time. In order to extend to the past, you need to use a negative value. Use time interval units like 5s, 1m, 3h. The default is 0.
- **Span end time shift -** Shift in the end time for the logs query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
- **Filter by Trace ID -** Toggle to append the trace ID to the logs query.
- **Filter by Span ID -** Toggle to append the span ID to the logs query.
![Trace to logs settings](/static/img/docs/explore/trace-to-logs-settings-8-2.png 'Screenshot of the trace to logs settings')
@ -146,8 +146,8 @@ datasources:
isDefault: false
jsonData:
tracesToLogs:
# Field with internal link pointing to a Loki data source in Grafana.
# datasourceUid value must match the `datasourceUid` value of the Loki data source.
# Field with internal link pointing to a logs data source in Grafana.
# datasourceUid value must match the `datasourceUid` value of the logs data source.
datasourceUid: 'loki'
tags: ['job', 'instance', 'pod', 'namespace']
mappedTags: [{ key: 'service.name', value: 'service' }]

View File

@ -27,15 +27,15 @@ To access Tempo settings, click the **Configuration** (gear) icon, then click **
> **Note:** This feature is available in Grafana 7.4+.
This is a configuration for the [trace to logs feature]({{< relref "../explore/trace-integration" >}}). Select target data source (at this moment limited to Loki data sources) and select which tags will be used in the logs query.
This is a configuration for the [trace to logs feature]({{< relref "../explore/trace-integration" >}}). Select target data source (at this moment limited to Loki or Splunk \[logs\] data sources) and select which tags will be used in the logs query.
- **Data source -** Target data source.
- **Tags -** The tags that will be used in the Loki query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
- **Map tag names -** When enabled, allows configuring how Tempo tag names map to Loki label names. For example, map `service.name` to `service`.
- **Span start time shift -** A shift in the start time for the Loki query based on the start time for the span. To extend the time to the past, use a negative value. You can use time units, for example, 5s, 1m, 3h. The default is 0.
- **Span end time shift -** Shift in the end time for the Loki query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
- **Filter by Trace ID -** Toggle to append the trace ID to the Loki query.
- **Filter by Span ID -** Toggle to append the span ID to the Loki query.
- **Tags -** The tags that will be used in the logs query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
- **Map tag names -** When enabled, allows configuring how Tempo tag names map to logs label names. For example, map `service.name` to `service`.
- **Span start time shift -** A shift in the start time for the logs query based on the start time for the span. To extend the time to the past, use a negative value. You can use time units, for example, 5s, 1m, 3h. The default is 0.
- **Span end time shift -** Shift in the end time for the logs query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
- **Filter by Trace ID -** Toggle to append the trace ID to the logs query.
- **Filter by Span ID -** Toggle to append the span ID to the logs query.
{{< figure src="/static/img/docs/explore/traces-to-logs-settings-8-2.png" class="docs-image--no-shadow" caption="Screenshot of the trace to logs settings" >}}

View File

@ -28,15 +28,15 @@ To access Zipkin settings, click the **Configuration** (gear) icon, then click *
> **Note:** This feature is available in Grafana 7.4+.
This is a configuration for the [trace to logs feature]({{< relref "../explore/trace-integration" >}}). Select target data source (at this moment limited to Loki data sources) and select which tags will be used in the logs query.
This is a configuration for the [trace to logs feature]({{< relref "../explore/trace-integration" >}}). Select target data source (at this moment limited to Loki or Splunk \[logs\] data sources) and select which tags will be used in the logs query.
- **Data source -** Target data source.
- **Tags -** The tags that will be used in the Loki query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
- **Map tag names -** When enabled, allows configuring how Zipkin tag names map to Loki label names. For example, map `service.name` to `service`.
- **Span start time shift -** Shift in the start time for the Loki query based on the span start time. In order to extend to the past, you need to use a negative value. Use time interval units like 5s, 1m, 3h. The default is 0.
- **Span end time shift -** Shift in the end time for the Loki query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
- **Filter by Trace ID -** Toggle to append the trace ID to the Loki query.
- **Filter by Span ID -** Toggle to append the span ID to the Loki query.
- **Tags -** The tags that will be used in the logs query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
- **Map tag names -** When enabled, allows configuring how Zipkin tag names map to logs label names. For example, map `service.name` to `service`.
- **Span start time shift -** Shift in the start time for the logs query based on the span start time. In order to extend to the past, you need to use a negative value. Use time interval units like 5s, 1m, 3h. The default is 0.
- **Span end time shift -** Shift in the end time for the logs query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
- **Filter by Trace ID -** Toggle to append the trace ID to the logs query.
- **Filter by Span ID -** Toggle to append the span ID to the logs query.
![Trace to logs settings](/static/img/docs/explore/trace-to-logs-settings-8-2.png 'Screenshot of the trace to logs settings')
@ -98,4 +98,4 @@ Here is an example JSON:
## Linking Trace ID from logs
You can link to Zipkin trace from logs in Loki by configuring a derived field with internal link. See [Loki documentation]({{< relref "loki#derived-fields" >}}) for details.
You can link to Zipkin trace from logs in Loki or Splunk by configuring a derived field with internal link. See [Loki documentation]({{< relref "loki#derived-fields" >}}) for details.

View File

@ -35,6 +35,8 @@ export interface DataSourcePickerProps {
variables?: boolean;
alerting?: boolean;
pluginId?: string;
/** If true,we show only DSs with logs; and if true, pluginId shouldnt be passed in */
logs?: boolean;
// If set to true and there is no value select will be empty, otherwise it will preselect default data source
noDefault?: boolean;
width?: number;
@ -123,12 +125,15 @@ export class DataSourcePicker extends PureComponent<DataSourcePickerProps, DataS
}
getDataSourceOptions() {
const { alerting, tracing, metrics, mixed, dashboard, variables, annotations, pluginId, type, filter } = this.props;
const { alerting, tracing, metrics, mixed, dashboard, variables, annotations, pluginId, type, filter, logs } =
this.props;
const options = this.dataSourceSrv
.getList({
alerting,
tracing,
metrics,
logs,
dashboard,
mixed,
variables,

View File

@ -46,6 +46,9 @@ export interface GetDataSourceListFilters {
/** Only return data sources that support tracing response */
tracing?: boolean;
/** Only return data sources that support logging response */
logs?: boolean;
/** Only return data sources that support annotations */
annotations?: boolean;

View File

@ -1,6 +1,7 @@
import { css } from '@emotion/css';
import {
DataSourceJsonData,
DataSourceInstanceSettings,
DataSourcePluginOptionsEditorProps,
GrafanaTheme,
KeyValue,
@ -44,12 +45,13 @@ export function TraceToLogsSettings({ options, onOptionsChange }: Props) {
<InlineField tooltip="The data source the trace is going to navigate to" label="Data source" labelWidth={26}>
<DataSourcePicker
inputId="trace-to-logs-data-source-picker"
pluginId="loki"
logs
current={options.jsonData.tracesToLogs?.datasourceUid}
noDefault={true}
width={40}
onChange={(ds) =>
onChange={(ds: DataSourceInstanceSettings) =>
updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToLogs', {
...options.jsonData.tracesToLogs,
datasourceUid: ds.uid,
tags: options.jsonData.tracesToLogs?.tags,
})

View File

@ -13,11 +13,11 @@ describe('createSpanLinkFactory', () => {
expect(createLink).not.toBeDefined();
});
describe('should return link', () => {
describe('should return loki link', () => {
beforeAll(() => {
setDataSourceSrv({
getInstanceSettings(uid: string): DataSourceInstanceSettings | undefined {
return { uid: 'loki1', name: 'loki1' } as any;
return { uid: 'loki1', name: 'loki1', type: 'loki' } as any;
},
} as any);
@ -215,14 +215,140 @@ describe('createSpanLinkFactory', () => {
expect(linkDef).toBeUndefined();
});
});
describe('should return splunk link', () => {
const splunkUID = 'splunkUID';
beforeAll(() => {
setDataSourceSrv({
getInstanceSettings(uid: string): DataSourceInstanceSettings | undefined {
return { uid: splunkUID, name: 'Splunk 8', type: 'grafana-splunk-datasource' } as any;
},
} as any);
setLinkSrv(new LinkSrv());
setTemplateSrv(new TemplateSrv());
});
it('the `query` keyword is used in the link rather than `expr` that loki uses', () => {
const createLink = setupSpanLinkFactory({
datasourceUid: splunkUID,
});
const linkDef = createLink!(createTraceSpan());
expect(linkDef!.href).toContain(`${encodeURIComponent('datasource":"Splunk 8","queries":[{"query"')}`);
expect(linkDef!.href).not.toContain(`${encodeURIComponent('datasource":"Splunk 8","queries":[{"expr"')}`);
});
it('automatically timeshifts the timerange by one second in a splunk query', () => {
const createLink = setupSpanLinkFactory({
datasourceUid: splunkUID,
});
const linkDef = createLink!(createTraceSpan());
expect(linkDef!.href).toContain(
`${encodeURIComponent('{"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"}')}`
);
expect(linkDef!.href).not.toContain(
`${encodeURIComponent('{"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:00.000Z"}')}`
);
});
it('formats query correctly if filterByTraceID and or filterBySpanID is true', () => {
const createLink = setupSpanLinkFactory({
datasourceUid: splunkUID,
filterByTraceID: true,
filterBySpanID: true,
});
expect(createLink).toBeDefined();
const linkDef = createLink!(createTraceSpan());
expect(linkDef!.href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"},"datasource":"Splunk 8","queries":[{"query":"cluster=\\"cluster1\\" hostname=\\"hostname1\\" \\"7946b05c2e2e4e5a\\" \\"6605c7b08e715d6c\\"","refId":""}],"panelsState":{}}'
)}`
);
});
it('should format one tag correctly', () => {
const createLink = setupSpanLinkFactory({
tags: ['ip'],
});
expect(createLink).toBeDefined();
const linkDef = createLink!(
createTraceSpan({
process: {
serviceName: 'service',
tags: [{ key: 'ip', value: '192.168.0.1' }],
},
})
);
expect(linkDef!.href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"},"datasource":"Splunk 8","queries":[{"query":"ip=\\"192.168.0.1\\"","refId":""}],"panelsState":{}}'
)}`
);
});
it('should format multiple tags correctly', () => {
const createLink = setupSpanLinkFactory({
tags: ['ip', 'hostname'],
});
expect(createLink).toBeDefined();
const linkDef = createLink!(
createTraceSpan({
process: {
serviceName: 'service',
tags: [
{ key: 'hostname', value: 'hostname1' },
{ key: 'ip', value: '192.168.0.1' },
],
},
})
);
expect(linkDef!.href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"},"datasource":"Splunk 8","queries":[{"query":"hostname=\\"hostname1\\" ip=\\"192.168.0.1\\"","refId":""}],"panelsState":{}}'
)}`
);
});
it('handles renamed tags', () => {
const createLink = setupSpanLinkFactory({
mapTagNamesEnabled: true,
mappedTags: [
{ key: 'service.name', value: 'service' },
{ key: 'k8s.pod.name', value: 'pod' },
],
});
expect(createLink).toBeDefined();
const linkDef = createLink!(
createTraceSpan({
process: {
serviceName: 'service',
tags: [
{ key: 'service.name', value: 'serviceName' },
{ key: 'k8s.pod.name', value: 'podName' },
],
},
})
);
expect(linkDef!.href).toBe(
`/explore?left=${encodeURIComponent(
'{"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"},"datasource":"Splunk 8","queries":[{"query":"service=\\"serviceName\\" pod=\\"podName\\"","refId":""}],"panelsState":{}}'
)}`
);
});
});
});
function setupSpanLinkFactory(options: Partial<TraceToLogsOptions> = {}) {
function setupSpanLinkFactory(options: Partial<TraceToLogsOptions> = {}, datasourceUid = 'lokiUid') {
const splitOpenFn = jest.fn();
return createSpanLinkFactory({
splitOpenFn,
traceToLogsOptions: {
datasourceUid: 'lokiUid',
datasourceUid,
...options,
},
});

View File

@ -1,9 +1,12 @@
import {
DataFrame,
DataLink,
DataQuery,
DataSourceInstanceSettings,
dateTime,
Field,
KeyValue,
LinkModel,
mapInternalLinkToExplore,
rangeUtil,
SplitOpen,
@ -70,6 +73,7 @@ function legacyCreateSpanLinkFactory(splitOpenFn: SplitOpen, traceToLogsOptions?
}
const dataSourceSettings = getDatasourceSrv().getInstanceSettings(traceToLogsOptions.datasourceUid);
const isSplunkDS = dataSourceSettings?.type === 'grafana-splunk-datasource';
if (!dataSourceSettings) {
return undefined;
@ -80,35 +84,37 @@ function legacyCreateSpanLinkFactory(splitOpenFn: SplitOpen, traceToLogsOptions?
// the moment. Issue is that the trace itself isn't clearly mapped to dataFrame (right now it's just a json blob
// inside a single field) so the dataLinks as config of that dataFrame abstraction breaks down a bit and we do
// it manually here instead of leaving it for the data source to supply the config.
let dataLink: DataLink<LokiQuery | DataQuery> | undefined = {} as DataLink<LokiQuery | DataQuery> | undefined;
let link: LinkModel<Field>;
const expr = getLokiQueryFromSpan(span, traceToLogsOptions);
if (!expr) {
return undefined;
switch (dataSourceSettings?.type) {
case 'loki':
dataLink = getLinkForLoki(span, traceToLogsOptions, dataSourceSettings);
if (!dataLink) {
return undefined;
}
break;
case 'grafana-splunk-datasource':
dataLink = getLinkForSplunk(span, traceToLogsOptions, dataSourceSettings);
break;
default:
return undefined;
}
const dataLink: DataLink<LokiQuery> = {
title: dataSourceSettings.name,
url: '',
internal: {
datasourceUid: dataSourceSettings.uid,
datasourceName: dataSourceSettings.name,
query: {
expr,
refId: '',
},
},
};
const link = mapInternalLinkToExplore({
link = mapInternalLinkToExplore({
link: dataLink,
internalLink: dataLink.internal!,
internalLink: dataLink?.internal!,
scopedVars: {},
range: getTimeRangeFromSpan(span, {
startMs: traceToLogsOptions.spanStartTimeShift
? rangeUtil.intervalToMs(traceToLogsOptions.spanStartTimeShift)
: 0,
endMs: traceToLogsOptions.spanEndTimeShift ? rangeUtil.intervalToMs(traceToLogsOptions.spanEndTimeShift) : 0,
}),
range: getTimeRangeFromSpan(
span,
{
startMs: traceToLogsOptions.spanStartTimeShift
? rangeUtil.intervalToMs(traceToLogsOptions.spanStartTimeShift)
: 0,
endMs: traceToLogsOptions.spanEndTimeShift ? rangeUtil.intervalToMs(traceToLogsOptions.spanEndTimeShift) : 0,
},
isSplunkDS
),
field: {} as Field,
onClickFn: splitOpenFn,
replaceVariables: getTemplateSrv().replace.bind(getTemplateSrv()),
@ -126,13 +132,11 @@ function legacyCreateSpanLinkFactory(splitOpenFn: SplitOpen, traceToLogsOptions?
* Default keys to use when there are no configured tags.
*/
const defaultKeys = ['cluster', 'hostname', 'namespace', 'pod'];
function getLokiQueryFromSpan(span: TraceSpan, options: TraceToLogsOptions): string | undefined {
function getLinkForLoki(span: TraceSpan, options: TraceToLogsOptions, dataSourceSettings: DataSourceInstanceSettings) {
const { tags: keys, filterByTraceID, filterBySpanID, mapTagNamesEnabled, mappedTags } = options;
// In order, try to use mapped tags -> tags -> default tags
const keysToCheck = mapTagNamesEnabled && mappedTags?.length ? mappedTags : keys?.length ? keys : defaultKeys;
// Build tag portion of query
const tags = [...span.process.tags, ...span.tags].reduce((acc, tag) => {
if (mapTagNamesEnabled) {
@ -152,17 +156,79 @@ function getLokiQueryFromSpan(span: TraceSpan, options: TraceToLogsOptions): str
if (!tags.length) {
return undefined;
}
let query = `{${tags.join(', ')}}`;
let expr = `{${tags.join(', ')}}`;
if (filterByTraceID && span.traceID) {
query += ` |="${span.traceID}"`;
expr += ` |="${span.traceID}"`;
}
if (filterBySpanID && span.spanID) {
query += ` |="${span.spanID}"`;
expr += ` |="${span.spanID}"`;
}
return query;
const dataLink: DataLink<LokiQuery> = {
title: dataSourceSettings.name,
url: '',
internal: {
datasourceUid: dataSourceSettings.uid,
datasourceName: dataSourceSettings.name,
query: {
expr: expr,
refId: '',
},
},
};
return dataLink;
}
function getLinkForSplunk(
span: TraceSpan,
options: TraceToLogsOptions,
dataSourceSettings: DataSourceInstanceSettings
) {
const { tags: keys, filterByTraceID, filterBySpanID, mapTagNamesEnabled, mappedTags } = options;
// In order, try to use mapped tags -> tags -> default tags
const keysToCheck = mapTagNamesEnabled && mappedTags?.length ? mappedTags : keys?.length ? keys : defaultKeys;
// Build tag portion of query
const tags = [...span.process.tags, ...span.tags].reduce((acc, tag) => {
if (mapTagNamesEnabled) {
const keyValue = (keysToCheck as KeyValue[]).find((keyValue: KeyValue) => keyValue.key === tag.key);
if (keyValue) {
acc.push(`${keyValue.value ? keyValue.value : keyValue.key}="${tag.value}"`);
}
} else {
if ((keysToCheck as string[]).includes(tag.key)) {
acc.push(`${tag.key}="${tag.value}"`);
}
}
return acc;
}, [] as string[]);
let query = '';
if (tags.length > 0) {
query += `${tags.join(' ')}`;
}
if (filterByTraceID && span.traceID) {
query += ` "${span.traceID}"`;
}
if (filterBySpanID && span.spanID) {
query += ` "${span.spanID}"`;
}
const dataLink: DataLink<DataQuery> = {
title: dataSourceSettings.name,
url: '',
internal: {
datasourceUid: dataSourceSettings.uid,
datasourceName: dataSourceSettings.name,
query: {
query: query,
refId: '',
},
},
} as DataLink<DataQuery>;
return dataLink;
}
/**
@ -170,18 +236,23 @@ function getLokiQueryFromSpan(span: TraceSpan, options: TraceToLogsOptions): str
*/
function getTimeRangeFromSpan(
span: TraceSpan,
timeShift: { startMs: number; endMs: number } = { startMs: 0, endMs: 0 }
timeShift: { startMs: number; endMs: number } = { startMs: 0, endMs: 0 },
isSplunkDS = false
): TimeRange {
const adjustedStartTime = Math.floor(span.startTime / 1000 + timeShift.startMs);
const from = dateTime(adjustedStartTime);
const spanEndMs = (span.startTime + span.duration) / 1000;
let adjustedEndTime = Math.floor(spanEndMs + timeShift.endMs);
// Because we can only pass milliseconds in the url we need to check if they equal.
// We need end time to be later than start time
if (adjustedStartTime === adjustedEndTime) {
// Splunk requires a time interval of >= 1s, rather than >=1ms like Loki timerange in below elseif block
if (isSplunkDS && adjustedEndTime - adjustedStartTime < 1000) {
adjustedEndTime = adjustedStartTime + 1000;
} else if (adjustedStartTime === adjustedEndTime) {
// Because we can only pass milliseconds in the url we need to check if they equal.
// We need end time to be later than start time
adjustedEndTime++;
}
const to = dateTime(adjustedEndTime);
// Beware that public/app/features/explore/state/main.ts SplitOpen fn uses the range from here. No matter what is in the url.

View File

@ -212,6 +212,9 @@ export class DatasourceSrv implements DataSourceService {
if (filters.tracing && !x.meta.tracing) {
return false;
}
if (filters.logs && x.meta.category !== 'logging' && !x.meta.logs) {
return false;
}
if (filters.annotations && !x.meta.annotations) {
return false;
}

View File

@ -113,7 +113,7 @@ class TempoQueryFieldComponent extends React.PureComponent<Props> {
</InlineField>
</InlineFieldRow>
{query.queryType === 'nativeSearch' && (
<p style={{ maxWidth: '65ch' }}>
<div style={{ maxWidth: '65ch' }}>
<Badge icon="rocket" text="Beta" color="blue" />
{config.featureToggles.tempoBackendSearch ? (
<>&nbsp;Tempo search is currently in beta.</>
@ -124,7 +124,7 @@ class TempoQueryFieldComponent extends React.PureComponent<Props> {
future!
</>
)}
</p>
</div>
)}
{query.queryType === 'search' && (
<SearchSection
@ -201,7 +201,6 @@ function SearchSection({ logsDatasourceUid, onChange, onRunQuery, query }: Searc
return (
<>
<InlineLabel>Tempo uses {ds.name} to find traces.</InlineLabel>
<LokiQueryField
datasource={ds}
onChange={onChange}

View File

@ -123,6 +123,7 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
settings.jsonData.derivedFields
?.filter((field) => field.datasourceUid === this.uid && field.matcherRegex)
.map((field) => field.matcherRegex) || [];
if (!traceLinkMatcher || traceLinkMatcher.length === 0) {
return throwError(
() =>