Tracing: add way to configure trace to logs start and end time (#34995)

* Tracing: add way to configure trace to logs start and end time

* Use the span's time by default

* Update docs

* Update time inputs to use strings instead of number

* Support negative values as well

* Add info about using negative value

* Don't round up Loki range

* Update docs/sources/datasources/jaeger.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Wording for doc

* Round adjusted start and end time

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

docs/sources/datasources/jaeger.md

@@ -32,8 +32,10 @@ This is a configuration for the [trace to logs feature]({{< relref "../explore/t
 
 - **Data source -** Target data source.
 - **Tags -** The tags that will be used in the Loki query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
+- **Span start time shift -** Shift in the start time for the Loki query based on the span start time. In order to extend to the past, you need to use a negative value.  Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
+- **Span end time shift -** Shift in the end time for the Loki query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
 
-![Trace to logs settings](/static/img/docs/explore/trace-to-logs-settings-7-4.png 'Screenshot of the trace to logs settings')
+![Trace to logs settings](/static/img/docs/explore/trace-to-logs-settings-8.png 'Screenshot of the trace to logs settings')
 
 ## Query traces
 

docs/sources/datasources/tempo.md

@@ -31,8 +31,10 @@ This is a configuration for the [trace to logs feature]({{< relref "../explore/t
 
 - **Data source -** Target data source.
 - **Tags -** The tags that will be used in the Loki query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
+- **Span start time shift -** Shift in the start time for the Loki query based on the span start time. In order to extend to the past, you need to use a negative value.  Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
+- **Span end time shift -** Shift in the end time for the Loki query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
 
-![Trace to logs settings](/static/img/docs/explore/trace-to-logs-settings-7-4.png 'Screenshot of the trace to logs settings')
+![Trace to logs settings](/static/img/docs/explore/trace-to-logs-settings-8.png 'Screenshot of the trace to logs settings')
 
 ## Query traces
 

docs/sources/datasources/zipkin.md

@@ -32,8 +32,10 @@ This is a configuration for the [trace to logs feature]({{< relref "../explore/t
 
 - **Data source -** Target data source.
 - **Tags -** The tags that will be used in the Loki query. Default is `'cluster', 'hostname', 'namespace', 'pod'`.
+- **Span start time shift -** Shift in the start time for the Loki query based on the span start time. In order to extend to the past, you need to use a negative value.  Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
+- **Span end time shift -** Shift in the end time for the Loki query based on the span end time. Time units can be used here, for example, 5s, 1m, 3h. The default is 0.
 
-![Trace to logs settings](/static/img/docs/explore/trace-to-logs-settings-7-4.png "Screenshot of the trace to logs settings")
+![Trace to logs settings](/static/img/docs/explore/trace-to-logs-settings-8.png 'Screenshot of the trace to logs settings')
 
 ## Query traces
 

packages/grafana-data/src/datetime/rangeutil.test.ts

@@ -41,6 +41,14 @@ describe('Range Utils', () => {
       expect(() => rangeUtil.describeInterval('123xyz')).toThrow();
       expect(() => rangeUtil.describeInterval('xyz')).toThrow();
     });
+
+    it('should be able to parse negative values as well', () => {
+      expect(rangeUtil.describeInterval('-50ms')).toEqual({
+        sec: 0.001,
+        type: 'ms',
+        count: -50,
+      });
+    });
   });
 
   describe('relativeToTimeRange', () => {

packages/grafana-data/src/datetime/rangeutil.ts

@@ -276,7 +276,7 @@ export function calculateInterval(range: TimeRange, resolution: number, lowLimit
   };
 }
 
-const interval_regex = /(\d+(?:\.\d+)?)(ms|[Mwdhmsy])/;
+const interval_regex = /(-?\d+(?:\.\d+)?)(ms|[Mwdhmsy])/;
 // histogram & trends
 const intervals_in_seconds = {
   y: 31536000,

public/app/core/components/TraceToLogsSettings.tsx

@@ -6,12 +6,14 @@ import {
   updateDatasourcePluginJsonDataOption,
 } from '@grafana/data';
 import { DataSourcePicker } from '@grafana/runtime';
-import { InlineField, InlineFieldRow, TagsInput, useStyles } from '@grafana/ui';
+import { InlineField, InlineFieldRow, Input, TagsInput, useStyles } from '@grafana/ui';
 import React from 'react';
 
 export interface TraceToLogsOptions {
   datasourceUid?: string;
   tags?: string[];
+  spanStartTimeShift?: string;
+  spanEndTimeShift?: string;
 }
 
 export interface TraceToLogsData extends DataSourceJsonData {
@@ -66,6 +68,50 @@ export function TraceToLogsSettings({ options, onOptionsChange }: Props) {
           />
         </InlineField>
       </InlineFieldRow>
+
+      <InlineFieldRow>
+        <InlineField
+          label="Span start time shift"
+          labelWidth={26}
+          grow
+          tooltip="Shifts the start time of the span. Default 0 (Time units can be used here, for example: 5s, 1m, 3h)"
+        >
+          <Input
+            type="text"
+            placeholder="1h"
+            width={40}
+            onChange={(v) =>
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToLogs', {
+                ...options.jsonData.tracesToLogs,
+                spanStartTimeShift: v.currentTarget.value,
+              })
+            }
+            value={options.jsonData.tracesToLogs?.spanStartTimeShift || ''}
+          />
+        </InlineField>
+      </InlineFieldRow>
+
+      <InlineFieldRow>
+        <InlineField
+          label="Span end time shift"
+          labelWidth={26}
+          grow
+          tooltip="Shifts the end time of the span. Default 0 Time units can be used here, for example: 5s, 1m, 3h"
+        >
+          <Input
+            type="text"
+            placeholder="1h"
+            width={40}
+            onChange={(v) =>
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToLogs', {
+                ...options.jsonData.tracesToLogs,
+                spanEndTimeShift: v.currentTarget.value,
+              })
+            }
+            value={options.jsonData.tracesToLogs?.spanEndTimeShift || ''}
+          />
+        </InlineField>
+      </InlineFieldRow>
     </div>
   );
 }

public/app/features/explore/TraceView/createSpanLink.test.ts

@@ -59,7 +59,7 @@ describe('createSpanLinkFactory', () => {
       } as any);
 
       expect(linkDef.href).toBe(
-        `/explore?left={"range":{"from":"20201014T000000","to":"20201014T010006"},"datasource":"loki1","queries":[{"expr":"{cluster=\\"cluster1\\", hostname=\\"hostname1\\"}","refId":""}]}`
+        `/explore?left={"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"},"datasource":"loki1","queries":[{"expr":"{cluster=\\"cluster1\\", hostname=\\"hostname1\\"}","refId":""}]}`
       );
     });
 
@@ -91,7 +91,7 @@ describe('createSpanLinkFactory', () => {
       } as any);
 
       expect(linkDef.href).toBe(
-        `/explore?left={"range":{"from":"20201014T000000","to":"20201014T010006"},"datasource":"loki1","queries":[{"expr":"{ip=\\"192.168.0.1\\"}","refId":""}]}`
+        `/explore?left={"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"},"datasource":"loki1","queries":[{"expr":"{ip=\\"192.168.0.1\\"}","refId":""}]}`
       );
     });
 
@@ -126,7 +126,44 @@ describe('createSpanLinkFactory', () => {
       } as any);
 
       expect(linkDef.href).toBe(
-        `/explore?left={"range":{"from":"20201014T000000","to":"20201014T010006"},"datasource":"loki1","queries":[{"expr":"{ip=\\"192.168.0.1\\", host=\\"host\\"}","refId":""}]}`
+        `/explore?left={"range":{"from":"2020-10-14T01:00:00.000Z","to":"2020-10-14T01:00:01.000Z"},"datasource":"loki1","queries":[{"expr":"{ip=\\"192.168.0.1\\", host=\\"host\\"}","refId":""}]}`
+      );
+    });
+
+    it('with adjusted start and end time', () => {
+      const splitOpenFn = jest.fn();
+      const createLink = createSpanLinkFactory(splitOpenFn, {
+        datasourceUid: 'lokiUid',
+        spanStartTimeShift: '1m',
+        spanEndTimeShift: '1m',
+      });
+
+      expect(createLink).toBeDefined();
+      const linkDef = createLink!({
+        startTime: new Date('2020-10-14T01:00:00Z').valueOf() * 1000,
+        duration: 1000 * 1000,
+        tags: [
+          {
+            key: 'host',
+            value: 'host',
+          },
+        ],
+        process: {
+          tags: [
+            {
+              key: 'hostname',
+              value: 'hostname1',
+            },
+            {
+              key: 'ip',
+              value: '192.168.0.1',
+            },
+          ],
+        } as any,
+      } as any);
+
+      expect(linkDef.href).toBe(
+        `/explore?left={"range":{"from":"2020-10-14T01:01:00.000Z","to":"2020-10-14T01:01:01.000Z"},"datasource":"loki1","queries":[{"expr":"{hostname=\\"hostname1\\"}","refId":""}]}`
       );
     });
   });

public/app/features/explore/TraceView/createSpanLink.tsx

@@ -1,12 +1,12 @@
-import { DataLink, dateTime, Field, mapInternalLinkToExplore, TimeRange } from '@grafana/data';
+import { DataLink, dateTime, Field, mapInternalLinkToExplore, rangeUtil, TimeRange } from '@grafana/data';
 import { getTemplateSrv } from '@grafana/runtime';
 import { Icon } from '@grafana/ui';
-import { SplitOpen } from 'app/types/explore';
+import { TraceSpan } from '@jaegertracing/jaeger-ui-components';
 import { TraceToLogsOptions } from 'app/core/components/TraceToLogsSettings';
 import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
+import { SplitOpen } from 'app/types/explore';
 import React from 'react';
 import { LokiQuery } from '../../../plugins/datasource/loki/types';
-import { TraceSpan } from '@jaegertracing/jaeger-ui-components';
 
 /**
  * This is a factory for the link creator. It returns the function mainly so it can return undefined in which case
@@ -48,7 +48,7 @@ export function createSpanLinkFactory(splitOpenFn: SplitOpen, traceToLogsOptions
       link: dataLink,
       internalLink: dataLink.internal!,
       scopedVars: {},
-      range: getTimeRangeFromSpan(span),
+      range: getTimeRangeFromSpan(span, traceToLogsOptions),
       field: {} as Field,
       onClickFn: splitOpenFn,
       replaceVariables: getTemplateSrv().replace.bind(getTemplateSrv()),
@@ -79,25 +79,32 @@ function getLokiQueryFromSpan(span: TraceSpan, keys?: string[]): string {
 }
 
 /**
- * Gets a time range from the span. Naively this could be just start and end time of the span but we also want some
+ * Gets a time range from the span.
- * buffer around that just so we do not miss some logs which may not have timestamps aligned with the span. Right
- * now the buffers are hardcoded which may be a bit weird for very short spans but at the same time, fractional buffers
- * with very short spans could mean microseconds and that could miss some logs relevant to that spans. In the future
- * something more intelligent should probably be implemented
  */
-function getTimeRangeFromSpan(span: TraceSpan): TimeRange {
+function getTimeRangeFromSpan(span: TraceSpan, traceToLogsOptions?: TraceToLogsOptions): TimeRange {
-  const from = dateTime(span.startTime / 1000 - 1000 * 60 * 60);
+  const adjustedStartTime = traceToLogsOptions?.spanStartTimeShift
+    ? Math.floor(span.startTime / 1000 + rangeUtil.intervalToMs(traceToLogsOptions.spanStartTimeShift))
+    : Math.floor(span.startTime / 1000);
+  const from = dateTime(adjustedStartTime);
   const spanEndMs = (span.startTime + span.duration) / 1000;
-  const to = dateTime(spanEndMs + 5 * 1000);
+  let adjustedEndTime = traceToLogsOptions?.spanEndTimeShift
+    ? Math.floor(spanEndMs + rangeUtil.intervalToMs(traceToLogsOptions.spanEndTimeShift))
+    : Math.floor(spanEndMs);
 
+  // Because we can only pass milliseconds in the url we need to check if they equal.
+  // We need end time to be later than start time
+  if (adjustedStartTime === adjustedEndTime) {
+    adjustedEndTime++;
+  }
+  const to = dateTime(adjustedEndTime);
+
+  // Beware that public/app/features/explore/state/main.ts SplitOpen fn uses the range from here. No matter what is in the url.
   return {
     from,
     to,
-    // Weirdly Explore does not handle ISO string which would have been the default stringification if passed as object
-    // and we have to use this custom format :( .
     raw: {
-      from: from.utc().format('YYYYMMDDTHHmmss'),
+      from,
-      to: to.utc().format('YYYYMMDDTHHmmss'),
+      to,
     },
   };
 }

public/app/plugins/datasource/loki/datasource.ts

@@ -184,14 +184,10 @@ export class LokiDatasource extends DataSourceApi<LokiQuery, LokiOptions> {
         this.adjustInterval((options as DataQueryRequest<LokiQuery>).intervalMs || 1000, rangeMs) / 1000;
       // We want to ceil to 3 decimal places
       const step = Math.ceil(adjustedInterval * 1000) / 1000;
-      const alignedTimes = {
-        start: startNs - (startNs % 1e9),
-        end: endNs + (1e9 - (endNs % 1e9)),
-      };
 
       range = {
-        start: alignedTimes.start,
+        start: startNs,
-        end: alignedTimes.end,
+        end: endNs,
         step,
       };
     }