Tracing: Trace to profiles (#76670)

* Update Tempo devenv to include profiles

* Update devenv to scrape profiles from local services

* Cleanup devenv

* Fix issue with flame graph

* Add width prop to ProfileTypeCascader

* Add trace to profiles settings

* Add new spanSelector API

* Add spanSelector to query editor

* Update span link query

* Conditionally show span link

* Combine profile and spanProfile query types and run specific query type in backend based on spanSelector presence

* Update placeholder

* Create feature toggle

* Remove spanProfile query type

* Cleanup

* Use feeature toggle

* Update feature toggle

* Update devenv

* Update devenv

* Tests

* Tests

* Profiles for this span

* Styling

* Types

* Update type check

* Tidier funcs

* Add config links from dataframe

* Remove time shift

* Update tests

* Update range in test

* Simplify span link logic

* Update default keys

* Update pyro link

* Use const

docs/sources/developers/kinds/composable/grafanapyroscope/dataquery/schema-reference.md

@@ -28,5 +28,6 @@ title: GrafanaPyroscopeDataQuery kind
 | `hide`          | boolean  | No       |         | true if query is disabled (ie should not be returned to the dashboard)<br/>Note this does not always imply that the query should not be executed since<br/>the results from a hidden query may be used as the input to other queries (SSE etc)          |
 | `maxNodes`      | integer  | No       |         | Sets the maximum number of nodes in the flamegraph.                                                                                                                                                                                                     |
 | `queryType`     | string   | No       |         | Specify the query flavor<br/>TODO make this required and give it a default                                                                                                                                                                              |
+| `spanSelector`  | string[] | No       |         | Specifies the query span selectors.                                                                                                                                                                                                                     |
 
 

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -132,6 +132,7 @@ Experimental features might be changed or removed without prior notice.
 | `grafanaAPIServer`                          | Enable Kubernetes API Server for Grafana resources                                                                                                                                                                                                                                |
 | `grafanaAPIServerWithExperimentalAPIs`      | Register experimental APIs with the k8s API server                                                                                                                                                                                                                                |
 | `featureToggleAdminPage`                    | Enable admin page for managing feature toggles from the Grafana front-end                                                                                                                                                                                                         |
+| `traceToProfiles`                           | Enables linking between traces and profiles                                                                                                                                                                                                                                       |
 | `permissionsFilterRemoveSubquery`           | Alternative permission filter implementation that does not use subqueries for fetching the dashboard folder                                                                                                                                                                       |
 | `influxdbSqlSupport`                        | Enable InfluxDB SQL query language support with new querying UI                                                                                                                                                                                                                   |
 | `angularDeprecationUI`                      | Display new Angular deprecation-related UI features                                                                                                                                                                                                                               |

go.mod

@@ -164,7 +164,7 @@ require (
 	github.com/go-openapi/validate v0.22.1 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // @grafana/backend-platform
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
-	github.com/golang/glog v1.1.0 // indirect
+	github.com/golang/glog v1.1.1 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // @grafana/backend-platform
 	github.com/google/btree v1.1.2 // indirect
@@ -286,7 +286,7 @@ require (
 
 require github.com/grafana/gofpdf v0.0.0-20231002120153-857cc45be447 // @grafana/sharing-squad
 
-require github.com/grafana/pyroscope/api v0.2.0 // @grafana/observability-traces-and-profiling
+require github.com/grafana/pyroscope/api v0.2.1 // @grafana/observability-traces-and-profiling
 
 require github.com/apache/arrow/go/v13 v13.0.0 // @grafana/observability-metrics
 
@@ -316,7 +316,7 @@ require (
 	github.com/cristalhq/jwt/v4 v4.0.2 // indirect
 	github.com/dave/jennifer v1.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/dgraph-io/ristretto v0.1.0 // indirect
+	github.com/dgraph-io/ristretto v0.1.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect

go.sum

@@ -992,6 +992,8 @@ github.com/dgraph-io/ristretto v0.0.1/go.mod h1:T40EBc7CJke8TkpiYfGGKAeFjSaxuFXh
 github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
 github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI=
 github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
@@ -1620,6 +1622,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU
 github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
 github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
+github.com/golang/glog v1.1.1 h1:jxpi2eWoU84wbX9iIEyAeeoac3FLuifZpY9tcNUD9kw=
+github.com/golang/glog v1.1.1/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -1833,6 +1837,8 @@ github.com/grafana/prometheus-alertmanager v0.25.1-0.20231027171310-70c52bf65758
 github.com/grafana/prometheus-alertmanager v0.25.1-0.20231027171310-70c52bf65758/go.mod h1:MmLemcsGjpbOwEeT3k7K+gnvIImXgkatCfVX6sOtx80=
 github.com/grafana/pyroscope/api v0.2.0 h1:TzOxL0s6SiaLEy944ZAKgHcx/JDRJXu4O8ObwkqR6p4=
 github.com/grafana/pyroscope/api v0.2.0/go.mod h1:nhH+xai9cYFgs6lMy/+L0pKj0d5yCMwji/QAiQFCP+U=
+github.com/grafana/pyroscope/api v0.2.1 h1:V/GSrwSN5HgA4Ijf/2SN9Sib55E/xObswaCMkdOOsxs=
+github.com/grafana/pyroscope/api v0.2.1/go.mod h1:vNO/Rym3pwNIN4y/f0ACrk5iR7DdWlsdfZGSZE+XChU=
 github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db h1:7aN5cccjIqCLTzedH7MZzRZt5/lsAHch6Z3L2ZGn5FA=
 github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db/go.mod h1:M5qHK+eWfAv8VR/265dIuEpL3fNfeC21tXXp9itM24A=
@@ -3531,6 +3537,7 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220829200755-d48e67d00261/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -112,6 +112,7 @@ export interface FeatureToggles {
   awsAsyncQueryCaching?: boolean;
   splitScopes?: boolean;
   azureMonitorDataplane?: boolean;
+  traceToProfiles?: boolean;
   permissionsFilterRemoveSubquery?: boolean;
   prometheusConfigOverhaulAuth?: boolean;
   configurableSchedulerTick?: boolean;

packages/grafana-schema/src/raw/composable/grafanapyroscope/dataquery/x/GrafanaPyroscopeDataQuery_types.gen.ts

@@ -34,9 +34,14 @@ export interface GrafanaPyroscopeDataQuery extends common.DataQuery {
    * Specifies the type of profile to query.
    */
   profileTypeId: string;
+  /**
+   * Specifies the query span selectors.
+   */
+  spanSelector?: Array<string>;
 }
 
 export const defaultGrafanaPyroscopeDataQuery: Partial<GrafanaPyroscopeDataQuery> = {
   groupBy: [],
   labelSelector: '{}',
+  spanSelector: [],
 };

pkg/services/featuremgmt/registry.go

@@ -653,6 +653,13 @@ var (
 			Owner:       grafanaPartnerPluginsSquad,
 			Expression:  "true", // on by default
 		},
+		{
+			Name:         "traceToProfiles",
+			Description:  "Enables linking between traces and profiles",
+			Stage:        FeatureStageExperimental,
+			FrontendOnly: true,
+			Owner:        grafanaObservabilityTracesAndProfilingSquad,
+		},
 		{
 			Name:        "permissionsFilterRemoveSubquery",
 			Description: "Alternative permission filter implementation that does not use subqueries for fetching the dashboard folder",

pkg/services/featuremgmt/toggles_gen.csv

@@ -93,6 +93,7 @@ featureToggleAdminPage,experimental,@grafana/grafana-operator-experience-squad,f
 awsAsyncQueryCaching,preview,@grafana/aws-datasources,false,false,false,false
 splitScopes,preview,@grafana/grafana-authnz-team,false,false,true,false
 azureMonitorDataplane,GA,@grafana/partner-datasources,false,false,false,false
+traceToProfiles,experimental,@grafana/observability-traces-and-profiling,false,false,false,true
 permissionsFilterRemoveSubquery,experimental,@grafana/backend-platform,false,false,false,false
 prometheusConfigOverhaulAuth,GA,@grafana/observability-metrics,false,false,false,false
 configurableSchedulerTick,experimental,@grafana/alerting-squad,false,false,true,false

pkg/services/featuremgmt/toggles_gen.go

@@ -383,6 +383,10 @@ const (
 	// Adds dataplane compliant frame metadata in the Azure Monitor datasource
 	FlagAzureMonitorDataplane = "azureMonitorDataplane"
 
+	// FlagTraceToProfiles
+	// Enables linking between traces and profiles
+	FlagTraceToProfiles = "traceToProfiles"
+
 	// FlagPermissionsFilterRemoveSubquery
 	// Alternative permission filter implementation that does not use subqueries for fetching the dashboard folder
 	FlagPermissionsFilterRemoveSubquery = "permissionsFilterRemoveSubquery"

pkg/tsdb/grafana-pyroscope-datasource/instance.go

@@ -31,6 +31,7 @@ type ProfilingClient interface {
 	LabelValues(ctx context.Context, label string) ([]string, error)
 	GetSeries(ctx context.Context, profileTypeID string, labelSelector string, start int64, end int64, groupBy []string, step float64) (*SeriesResponse, error)
 	GetProfile(ctx context.Context, profileTypeID string, labelSelector string, start int64, end int64, maxNodes *int64) (*ProfileResponse, error)
+	GetSpanProfile(ctx context.Context, profileTypeID string, labelSelector string, spanSelector []string, start int64, end int64, maxNodes *int64) (*ProfileResponse, error)
 }
 
 // PyroscopeDatasource is a datasource for querying application performance profiles.

pkg/tsdb/grafana-pyroscope-datasource/kinds/dataquery/types_dataquery_gen.go

@@ -79,6 +79,9 @@ type GrafanaPyroscopeDataQuery struct {
 	// In server side expressions, the refId is used as a variable name to identify results.
 	// By default, the UI will assign A->Z; however setting meaningful names may be useful.
 	RefId string `json:"refId"`
+
+	// Specifies the query span selectors.
+	SpanSelector []string `json:"spanSelector,omitempty"`
 }
 
 // PyroscopeQueryType defines model for PyroscopeQueryType.

pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient.go

@@ -175,8 +175,41 @@ func (c *PyroscopeClient) GetProfile(ctx context.Context, profileTypeID, labelSe
 		return nil, nil
 	}
 
-	levels := make([]*Level, len(resp.Msg.Flamegraph.Levels))
+	return profileQuery(ctx, err, span, resp.Msg.Flamegraph, profileTypeID)
-	for i, level := range resp.Msg.Flamegraph.Levels {
+}
+
+func (c *PyroscopeClient) GetSpanProfile(ctx context.Context, profileTypeID, labelSelector string, spanSelector []string, start, end int64, maxNodes *int64) (*ProfileResponse, error) {
+	ctx, span := tracing.DefaultTracer().Start(ctx, "datasource.pyroscope.GetSpanProfile", trace.WithAttributes(attribute.String("profileTypeID", profileTypeID), attribute.String("labelSelector", labelSelector), attribute.String("spanSelector", strings.Join(spanSelector, ","))))
+	defer span.End()
+	req := &connect.Request[querierv1.SelectMergeSpanProfileRequest]{
+		Msg: &querierv1.SelectMergeSpanProfileRequest{
+			ProfileTypeID: profileTypeID,
+			LabelSelector: labelSelector,
+			SpanSelector:  spanSelector,
+			Start:         start,
+			End:           end,
+			MaxNodes:      maxNodes,
+		},
+	}
+
+	resp, err := c.connectClient.SelectMergeSpanProfile(ctx, req)
+	if err != nil {
+		span.RecordError(err)
+		span.SetStatus(codes.Error, err.Error())
+		return nil, err
+	}
+
+	if resp.Msg.Flamegraph == nil {
+		// Not an error, can happen when querying data oout of range.
+		return nil, nil
+	}
+
+	return profileQuery(ctx, err, span, resp.Msg.Flamegraph, profileTypeID)
+}
+
+func profileQuery(ctx context.Context, err error, span trace.Span, flamegraph *querierv1.FlameGraph, profileTypeID string) (*ProfileResponse, error) {
+	levels := make([]*Level, len(flamegraph.Levels))
+	for i, level := range flamegraph.Levels {
 		levels[i] = &Level{
 			Values: level.Values,
 		}
@@ -184,10 +217,10 @@ func (c *PyroscopeClient) GetProfile(ctx context.Context, profileTypeID, labelSe
 
 	return &ProfileResponse{
 		Flamebearer: &Flamebearer{
-			Names:   resp.Msg.Flamegraph.Names,
+			Names:   flamegraph.Names,
 			Levels:  levels,
-			Total:   resp.Msg.Flamegraph.Total,
+			Total:   flamegraph.Total,
-			MaxSelf: resp.Msg.Flamegraph.MaxSelf,
+			MaxSelf: flamegraph.MaxSelf,
 		},
 		Units: getUnits(profileTypeID),
 	}, nil

pkg/tsdb/grafana-pyroscope-datasource/pyroscopeClient_test.go

@@ -129,3 +129,7 @@ func (f *FakePyroscopeConnectClient) SelectSeries(ctx context.Context, req *conn
 func (f *FakePyroscopeConnectClient) SelectMergeProfile(ctx context.Context, c *connect.Request[querierv1.SelectMergeProfileRequest]) (*connect.Response[googlev1.Profile], error) {
 	panic("implement me")
 }
+
+func (f *FakePyroscopeConnectClient) SelectMergeSpanProfile(ctx context.Context, c *connect.Request[querierv1.SelectMergeSpanProfileRequest]) (*connect.Response[querierv1.SelectMergeSpanProfileResponse], error) {
+	panic("implement me")
+}

pkg/tsdb/grafana-pyroscope-datasource/query.go

@@ -98,18 +98,32 @@ func (d *PyroscopeDatasource) query(ctx context.Context, pCtx backend.PluginCont
 
 	if query.QueryType == queryTypeProfile || query.QueryType == queryTypeBoth {
 		g.Go(func() error {
-			logger.Debug("Calling GetProfile", "queryModel", qm, "function", logEntrypoint())
+			var profileResp *ProfileResponse
-			prof, err := d.client.GetProfile(gCtx, qm.ProfileTypeId, qm.LabelSelector, query.TimeRange.From.UnixMilli(), query.TimeRange.To.UnixMilli(), qm.MaxNodes)
+			if len(qm.SpanSelector) > 0 {
-			if err != nil {
+				logger.Debug("Calling GetSpanProfile", "queryModel", qm, "function", logEntrypoint())
-				span.RecordError(err)
+				prof, err := d.client.GetSpanProfile(gCtx, qm.ProfileTypeId, qm.LabelSelector, qm.SpanSelector, query.TimeRange.From.UnixMilli(), query.TimeRange.To.UnixMilli(), qm.MaxNodes)
-				span.SetStatus(codes.Error, err.Error())
+				if err != nil {
-				logger.Error("Error GetProfile()", "err", err, "function", logEntrypoint())
+					span.RecordError(err)
-				return err
+					span.SetStatus(codes.Error, err.Error())
+					logger.Error("Error GetSpanProfile()", "err", err, "function", logEntrypoint())
+					return err
+				}
+				profileResp = prof
+			} else {
+				logger.Debug("Calling GetProfile", "queryModel", qm, "function", logEntrypoint())
+				prof, err := d.client.GetProfile(gCtx, qm.ProfileTypeId, qm.LabelSelector, query.TimeRange.From.UnixMilli(), query.TimeRange.To.UnixMilli(), qm.MaxNodes)
+				if err != nil {
+					span.RecordError(err)
+					span.SetStatus(codes.Error, err.Error())
+					logger.Error("Error GetProfile()", "err", err, "function", logEntrypoint())
+					return err
+				}
+				profileResp = prof
 			}
 
 			var frame *data.Frame
-			if prof != nil {
+			if profileResp != nil {
-				frame = responseToDataFrames(prof)
+				frame = responseToDataFrames(profileResp)
 
 				// If query called with streaming on then return a channel
 				// to subscribe on a client-side and consume updates from a plugin.

pkg/tsdb/grafana-pyroscope-datasource/query_test.go

@@ -312,6 +312,22 @@ func (f *FakeClient) GetProfile(ctx context.Context, profileTypeID, labelSelecto
 	}, nil
 }
 
+func (f *FakeClient) GetSpanProfile(ctx context.Context, profileTypeID, labelSelector string, spanSelector []string, start, end int64, maxNodes *int64) (*ProfileResponse, error) {
+	return &ProfileResponse{
+		Flamebearer: &Flamebearer{
+			Names: []string{"foo", "bar", "baz"},
+			Levels: []*Level{
+				{Values: []int64{0, 10, 0, 0}},
+				{Values: []int64{0, 9, 0, 1}},
+				{Values: []int64{0, 8, 8, 2}},
+			},
+			Total:   100,
+			MaxSelf: 56,
+		},
+		Units: "count",
+	}, nil
+}
+
 func (f *FakeClient) GetSeries(ctx context.Context, profileTypeID, labelSelector string, start, end int64, groupBy []string, step float64) (*SeriesResponse, error) {
 	f.Args = []any{profileTypeID, labelSelector, start, end, groupBy, step}
 	return &SeriesResponse{

public/app/core/components/TraceToProfiles/TraceToProfilesSettings.test.tsx

@@ -0,0 +1,56 @@
+import { render, screen, waitFor } from '@testing-library/react';
+import React from 'react';
+
+import { DataSourceInstanceSettings, DataSourceSettings } from '@grafana/data';
+import { DataSourceSrv, setDataSourceSrv } from '@grafana/runtime';
+
+import { TraceToProfilesData, TraceToProfilesSettings } from './TraceToProfilesSettings';
+
+const defaultOption: DataSourceSettings<TraceToProfilesData> = {
+  jsonData: {
+    tracesToProfilesV2: {
+      datasourceUid: 'profiling1_uid',
+      tags: [{ key: 'someTag', value: 'newName' }],
+      spanStartTimeShift: '1m',
+      spanEndTimeShift: '1m',
+      customQuery: true,
+      query: '{${__tags}}',
+    },
+  },
+} as unknown as DataSourceSettings<TraceToProfilesData>;
+
+const pyroSettings = {
+  uid: 'profiling1_uid',
+  name: 'profiling1',
+  type: 'grafana-pyroscope-datasource',
+  meta: { info: { logos: { small: '' } } },
+} as unknown as DataSourceInstanceSettings;
+
+describe('TraceToProfilesSettings', () => {
+  beforeAll(() => {
+    setDataSourceSrv({
+      getList() {
+        return [pyroSettings];
+      },
+      getInstanceSettings() {
+        return pyroSettings;
+      },
+    } as unknown as DataSourceSrv);
+  });
+
+  it('should render without error', () => {
+    waitFor(() => {
+      expect(() =>
+        render(<TraceToProfilesSettings options={defaultOption} onOptionsChange={() => {}} />)
+      ).not.toThrow();
+    });
+  });
+
+  it('should render all options', () => {
+    render(<TraceToProfilesSettings options={defaultOption} onOptionsChange={() => {}} />);
+    expect(screen.getByText('Select data source')).toBeInTheDocument();
+    expect(screen.getByText('Tags')).toBeInTheDocument();
+    expect(screen.getByText('Profile type')).toBeInTheDocument();
+    expect(screen.getByText('Use custom query')).toBeInTheDocument();
+  });
+});

public/app/core/components/TraceToProfiles/TraceToProfilesSettings.tsx

@@ -0,0 +1,187 @@
+import { css } from '@emotion/css';
+import React, { useEffect, useMemo, useState } from 'react';
+import { useAsync } from 'react-use';
+
+import {
+  DataSourceJsonData,
+  DataSourceInstanceSettings,
+  DataSourcePluginOptionsEditorProps,
+  updateDatasourcePluginJsonDataOption,
+} from '@grafana/data';
+import { ConfigSection } from '@grafana/experimental';
+import { getDataSourceSrv } from '@grafana/runtime';
+import { InlineField, InlineFieldRow, Input, InlineSwitch } from '@grafana/ui';
+import { ConfigDescriptionLink } from 'app/core/components/ConfigDescriptionLink';
+import { DataSourcePicker } from 'app/features/datasources/components/picker/DataSourcePicker';
+import { ProfileTypesCascader } from 'app/plugins/datasource/grafana-pyroscope-datasource/QueryEditor/ProfileTypesCascader';
+import { PyroscopeDataSource } from 'app/plugins/datasource/grafana-pyroscope-datasource/datasource';
+import { ProfileTypeMessage } from 'app/plugins/datasource/grafana-pyroscope-datasource/types';
+
+import { TagMappingInput } from '../TraceToLogs/TagMappingInput';
+export interface TraceToProfilesOptions {
+  datasourceUid?: string;
+  tags?: Array<{ key: string; value?: string }>;
+  query?: string;
+  profileTypeId?: string;
+  customQuery: boolean;
+}
+
+export interface TraceToProfilesData extends DataSourceJsonData {
+  tracesToProfiles?: TraceToProfilesOptions;
+}
+
+interface Props extends DataSourcePluginOptionsEditorProps<TraceToProfilesData> {}
+
+export function TraceToProfilesSettings({ options, onOptionsChange }: Props) {
+  const supportedDataSourceTypes = useMemo(() => ['grafana-pyroscope-datasource'], []);
+
+  const [profileTypes, setProfileTypes] = useState<ProfileTypeMessage[]>([]);
+  const profileTypesPlaceholder = useMemo(() => {
+    let placeholder = profileTypes.length === 0 ? 'No profile types found' : 'Select profile type';
+    if (!options.jsonData.tracesToProfiles?.datasourceUid) {
+      placeholder = 'Please select profiling data source';
+    }
+    return placeholder;
+  }, [options.jsonData.tracesToProfiles?.datasourceUid, profileTypes]);
+
+  const { value: dataSource } = useAsync(async () => {
+    return await getDataSourceSrv().get(options.jsonData.tracesToProfiles?.datasourceUid);
+  }, [options.jsonData.tracesToProfiles?.datasourceUid]);
+
+  useEffect(() => {
+    if (
+      dataSource &&
+      dataSource instanceof PyroscopeDataSource &&
+      supportedDataSourceTypes.includes(dataSource.type) &&
+      dataSource.uid === options.jsonData.tracesToProfiles?.datasourceUid
+    ) {
+      dataSource.getProfileTypes().then((profileTypes) => {
+        setProfileTypes(profileTypes);
+      });
+    } else {
+      setProfileTypes([]);
+    }
+  }, [dataSource, onOptionsChange, options, supportedDataSourceTypes]);
+
+  return (
+    <div className={css({ width: '100%' })}>
+      <InlineFieldRow>
+        <InlineField
+          tooltip="The profiles data source the trace is going to navigate to"
+          label="Data source"
+          labelWidth={26}
+        >
+          <DataSourcePicker
+            inputId="trace-to-profiles-data-source-picker"
+            filter={(ds) => supportedDataSourceTypes.includes(ds.type)}
+            current={options.jsonData.tracesToProfiles?.datasourceUid}
+            noDefault={true}
+            width={40}
+            onChange={(ds: DataSourceInstanceSettings) => {
+              console.log(options.jsonData.tracesToProfiles, ds);
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToProfiles', {
+                ...options.jsonData.tracesToProfiles,
+                datasourceUid: ds.uid,
+              });
+            }}
+          />
+        </InlineField>
+      </InlineFieldRow>
+
+      <InlineFieldRow>
+        <InlineField
+          tooltip="Tags that will be used in the query. Default tags: 'service.name', 'service.namespace'"
+          label="Tags"
+          labelWidth={26}
+        >
+          <TagMappingInput
+            values={options.jsonData.tracesToProfiles?.tags ?? []}
+            onChange={(v) => {
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToProfiles', {
+                ...options.jsonData.tracesToProfiles,
+                tags: v,
+              });
+            }}
+          />
+        </InlineField>
+      </InlineFieldRow>
+
+      <InlineFieldRow>
+        <InlineField tooltip="Profile type that will be used in the query" label="Profile type" labelWidth={26}>
+          <ProfileTypesCascader
+            profileTypes={profileTypes}
+            placeholder={profileTypesPlaceholder}
+            initialProfileTypeId={options.jsonData.tracesToProfiles?.profileTypeId}
+            onChange={(val) => {
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToProfiles', {
+                ...options.jsonData.tracesToProfiles,
+                profileTypeId: val,
+              });
+            }}
+            width={40}
+          />
+        </InlineField>
+      </InlineFieldRow>
+
+      <InlineFieldRow>
+        <InlineField
+          tooltip="Use a custom query with the possibility to interpolate variables from the trace or span"
+          label="Use custom query"
+          labelWidth={26}
+        >
+          <InlineSwitch
+            id={'profilesCustomQuerySwitch'}
+            value={options.jsonData.tracesToProfiles?.customQuery}
+            onChange={(event: React.SyntheticEvent<HTMLInputElement>) =>
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToProfiles', {
+                ...options.jsonData.tracesToProfiles,
+                customQuery: event.currentTarget.checked,
+              })
+            }
+          />
+        </InlineField>
+      </InlineFieldRow>
+
+      {options.jsonData.tracesToProfiles?.customQuery && (
+        <InlineField
+          label="Query"
+          labelWidth={26}
+          tooltip="The query that will run when navigating from a trace to profiles data source. Interpolate tags using the `$__tags` keyword"
+          grow
+        >
+          <Input
+            label="Query"
+            type="text"
+            allowFullScreen
+            value={options.jsonData.tracesToProfiles?.query || ''}
+            onChange={(e) =>
+              updateDatasourcePluginJsonDataOption({ onOptionsChange, options }, 'tracesToProfiles', {
+                ...options.jsonData.tracesToProfiles,
+                query: e.currentTarget.value,
+              })
+            }
+          />
+        </InlineField>
+      )}
+    </div>
+  );
+}
+
+export const TraceToProfilesSection = ({ options, onOptionsChange }: DataSourcePluginOptionsEditorProps) => {
+  return (
+    <ConfigSection
+      title="Trace to profiles"
+      description={
+        <ConfigDescriptionLink
+          description="Navigate from a trace span to the selected data source's profiles."
+          suffix={`${options.type}/#trace-to-profiles`}
+          feature="trace to profiles"
+        />
+      }
+      isCollapsible={true}
+      isInitiallyOpen={true}
+    >
+      <TraceToProfilesSettings options={options} onOptionsChange={onOptionsChange} />
+    </ConfigSection>
+  );
+};

public/app/features/explore/TraceView/TraceView.tsx

@@ -20,6 +20,7 @@ import { DataQuery } from '@grafana/schema';
 import { useStyles2 } from '@grafana/ui';
 import { getTraceToLogsOptions } from 'app/core/components/TraceToLogs/TraceToLogsSettings';
 import { TraceToMetricsData } from 'app/core/components/TraceToMetrics/TraceToMetricsSettings';
+import { TraceToProfilesData } from 'app/core/components/TraceToProfiles/TraceToProfilesSettings';
 import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
 import { getTimeZone } from 'app/features/profile/state/selectors';
 import { TempoQuery } from 'app/plugins/datasource/tempo/types';
@@ -132,6 +133,8 @@ export function TraceView(props: Props) {
   const traceToLogsOptions = getTraceToLogsOptions(instanceSettings?.jsonData);
   const traceToMetrics: TraceToMetricsData | undefined = instanceSettings?.jsonData;
   const traceToMetricsOptions = traceToMetrics?.tracesToMetrics;
+  const traceToProfilesData: TraceToProfilesData | undefined = instanceSettings?.jsonData;
+  const traceToProfilesOptions = traceToProfilesData?.tracesToProfiles;
   const spanBarOptions: SpanBarOptionsData | undefined = instanceSettings?.jsonData;
 
   const createSpanLink = useMemo(
@@ -141,6 +144,7 @@ export function TraceView(props: Props) {
         splitOpenFn: props.splitOpenFn!,
         traceToLogsOptions,
         traceToMetricsOptions,
+        traceToProfilesOptions,
         dataFrame: props.dataFrames[0],
         createFocusSpanLink,
         trace: traceProp,
@@ -149,6 +153,7 @@ export function TraceView(props: Props) {
       props.splitOpenFn,
       traceToLogsOptions,
       traceToMetricsOptions,
+      traceToProfilesOptions,
       props.dataFrames,
       createFocusSpanLink,
       traceProp,

public/app/features/explore/TraceView/components/TraceTimelineViewer/SpanDetail/index.tsx

@@ -17,16 +17,17 @@ import { SpanStatusCode } from '@opentelemetry/api';
 import cx from 'classnames';
 import React from 'react';
 
-import { dateTimeFormat, GrafanaTheme2, LinkModel, TimeZone } from '@grafana/data';
+import { dateTimeFormat, GrafanaTheme2, IconName, LinkModel, TimeZone } from '@grafana/data';
 import { config, locationService, reportInteraction } from '@grafana/runtime';
 import { DataLinkButton, Icon, TextArea, useStyles2 } from '@grafana/ui';
+import { RelatedProfilesTitle } from 'app/plugins/datasource/tempo/resultTransformer';
 
 import { autoColor } from '../../Theme';
 import { Divider } from '../../common/Divider';
 import LabeledList from '../../common/LabeledList';
 import { KIND, LIBRARY_NAME, LIBRARY_VERSION, STATUS, STATUS_MESSAGE, TRACE_STATE } from '../../constants/span';
 import { SpanLinkFunc, TNil } from '../../types';
-import { SpanLinkType } from '../../types/links';
+import { SpanLinkDef, SpanLinkType } from '../../types/links';
 import { TraceKeyValuePair, TraceLink, TraceLog, TraceSpan, TraceSpanReference } from '../../types/trace';
 import { uAlignIcon, ubM0, ubMb1, ubMy1, ubTxRightAlign } from '../../uberUtilityStyles';
 import { TopOfViewRefType } from '../VirtualizedTraceView';
@@ -241,39 +242,50 @@ export default function SpanDetail(props: SpanDetailProps) {
 
   const styles = useStyles2(getStyles);
 
-  let logLinkButton: JSX.Element | undefined = undefined;
+  const createLinkButton = (link: SpanLinkDef, type: SpanLinkType, title: string, icon: IconName) => {
+    return (
+      <DataLinkButton
+        link={{
+          ...link,
+          title: title,
+          target: '_blank',
+          origin: link.field,
+          onClick: (event: React.MouseEvent) => {
+            // DataLinkButton assumes if you provide an onClick event you would want to prevent default behavior like navigation
+            // In this case, if an onClick is not defined, restore navigation to the provided href while keeping the tracking
+            // this interaction will not be tracked with link right clicks
+            reportInteraction('grafana_traces_trace_view_span_link_clicked', {
+              datasourceType: datasourceType,
+              grafana_version: config.buildInfo.version,
+              type,
+              location: 'spanDetails',
+            });
+
+            if (link.onClick) {
+              link.onClick?.(event);
+            } else {
+              locationService.push(link.href);
+            }
+          },
+        }}
+        buttonProps={{ icon }}
+      />
+    );
+  };
+
+  let logLinkButton: JSX.Element | null = null;
+  let profileLinkButton: JSX.Element | null = null;
   if (createSpanLink) {
     const links = createSpanLink(span);
-    const logLinks = links?.filter((link) => link.type === SpanLinkType.Logs);
+    const logsLink = links?.filter((link) => link.type === SpanLinkType.Logs);
-    if (links && logLinks && logLinks.length > 0) {
+    if (links && logsLink && logsLink.length > 0) {
-      logLinkButton = (
+      logLinkButton = createLinkButton(logsLink[0], SpanLinkType.Logs, 'Logs for this span', 'gf-logs');
-        <DataLinkButton
+    }
-          link={{
+    const profilesLink = links?.filter(
-            ...logLinks[0],
+      (link) => link.type === SpanLinkType.Profiles && link.title === RelatedProfilesTitle
-            title: 'Logs for this span',
+    );
-            target: '_blank',
+    if (links && profilesLink && profilesLink.length > 0) {
-            origin: logLinks[0].field,
+      profileLinkButton = createLinkButton(profilesLink[0], SpanLinkType.Profiles, 'Profiles for this span', 'link');
-            onClick: (event: React.MouseEvent) => {
-              // DataLinkButton assumes if you provide an onClick event you would want to prevent default behavior like navigation
-              // In this case, if an onClick is not defined, restore navigation to the provided href while keeping the tracking
-              // this interaction will not be tracked with link right clicks
-              reportInteraction('grafana_traces_trace_view_span_link_clicked', {
-                datasourceType: datasourceType,
-                grafana_version: config.buildInfo.version,
-                type: 'log',
-                location: 'spanDetails',
-              });
-
-              if (logLinks?.[0].onClick) {
-                logLinks?.[0].onClick?.(event);
-              } else {
-                locationService.push(logLinks?.[0].href);
-              }
-            },
-          }}
-          buttonProps={{ icon: 'gf-logs' }}
-        />
-      );
     }
   }
 
@@ -286,7 +298,8 @@ export default function SpanDetail(props: SpanDetailProps) {
           <LabeledList className={ubTxRightAlign} divider={true} items={overviewItems} />
         </div>
       </div>
-      {logLinkButton}
+      <span style={{ marginRight: '10px' }}>{logLinkButton}</span>
+      {profileLinkButton}
       <Divider className={ubMy1} type={'horizontal'} />
       <div>
         <div>

public/app/features/explore/TraceView/components/types/links.ts

@@ -8,6 +8,7 @@ export enum SpanLinkType {
   Logs = 'log',
   Traces = 'trace',
   Metrics = 'metric',
+  Profiles = 'profile',
   Unknown = 'unknown',
 }
 

public/app/features/explore/TraceView/createSpanLink.test.ts

@@ -5,8 +5,9 @@ import {
   SupportedTransformationType,
   DataLinkConfigOrigin,
   FieldType,
+  DataFrame,
 } from '@grafana/data';
-import { DataSourceSrv, setDataSourceSrv, setTemplateSrv } from '@grafana/runtime';
+import { config, DataSourceSrv, setDataSourceSrv, setTemplateSrv } from '@grafana/runtime';
 import { TraceToMetricsOptions } from 'app/core/components/TraceToMetrics/TraceToMetricsSettings';
 import { DatasourceSrv } from 'app/features/plugins/datasource_srv';
 
@@ -19,7 +20,43 @@ import { SpanLinkType } from './components/types/links';
 import { createSpanLinkFactory } from './createSpanLink';
 
 const dummyTraceData = { duration: 10, traceID: 'trace1', traceName: 'test trace' } as unknown as Trace;
-const dummyDataFrame = createDataFrame({ fields: [{ name: 'traceId', values: ['trace1'] }] });
+const dummyDataFrame = createDataFrame({
+  fields: [
+    { name: 'traceId', values: ['trace1'] },
+    { name: 'spanID', values: ['testSpanId'] },
+  ],
+});
+const dummyDataFrameForProfiles = createDataFrame({
+  fields: [
+    { name: 'traceId', values: ['trace1'] },
+    { name: 'spanID', values: ['testSpanId'] },
+    {
+      name: 'tags',
+      config: {
+        links: [
+          {
+            internal: {
+              query: {
+                labelSelector: '{${__tags}}',
+                groupBy: [],
+                profileTypeId: '',
+                queryType: 'profile',
+                spanSelector: ['${__span.spanId}'],
+                refId: '',
+              },
+              datasourceUid: 'pyroscopeUid',
+              datasourceName: 'pyroscope',
+            },
+            url: '',
+            title: 'Test',
+            origin: DataLinkConfigOrigin.Datasource,
+          },
+        ],
+      },
+      values: [{ key: 'test', value: 'test' }],
+    },
+  ],
+});
 
 jest.mock('app/core/services/context_srv', () => ({
   contextSrv: {
@@ -1229,6 +1266,206 @@ describe('createSpanLinkFactory', () => {
       );
     });
   });
+
+  describe('should return pyroscope link', () => {
+    beforeAll(() => {
+      setDataSourceSrv({
+        getInstanceSettings() {
+          return {
+            uid: 'pyroscopeUid',
+            name: 'pyroscope',
+            type: 'grafana-pyroscope-datasource',
+          } as unknown as DataSourceInstanceSettings;
+        },
+      } as unknown as DataSourceSrv);
+
+      setLinkSrv(new LinkSrv());
+      setTemplateSrv(new TemplateSrv());
+      config.featureToggles.traceToProfiles = true;
+    });
+
+    it('with default keys when tags not configured', () => {
+      const createLink = setupSpanLinkFactory({}, '', dummyDataFrameForProfiles);
+      expect(createLink).toBeDefined();
+      const links = createLink!(createTraceSpan());
+      const linkDef = links?.[0];
+      expect(linkDef).toBeDefined();
+      expect(linkDef?.type).toBe(SpanLinkType.Profiles);
+      expect(linkDef!.href).toBe(
+        `/explore?left=${encodeURIComponent(
+          '{"range":{"from":"1602637140000","to":"1602637261000"},"datasource":"pyroscopeUid","queries":[{"labelSelector":"{service_namespace=\\"namespace1\\"}","groupBy":[],"profileTypeId":"","queryType":"profile","spanSelector":["6605c7b08e715d6c"],"refId":""}]}'
+        )}`
+      );
+    });
+
+    it('with tags that passed in and without tags that are not in the span', () => {
+      const createLink = setupSpanLinkFactory(
+        {
+          tags: [{ key: 'ip' }, { key: 'newTag' }],
+        },
+        '',
+        dummyDataFrameForProfiles
+      );
+      expect(createLink).toBeDefined();
+      const links = createLink!(
+        createTraceSpan({
+          process: {
+            serviceName: 'service',
+            tags: [
+              { key: 'hostname', value: 'hostname1' },
+              { key: 'ip', value: '192.168.0.1' },
+            ],
+          },
+        })
+      );
+      const linkDef = links?.[0];
+      expect(linkDef).toBeDefined();
+      expect(linkDef?.type).toBe(SpanLinkType.Profiles);
+      expect(linkDef!.href).toBe(
+        `/explore?left=${encodeURIComponent(
+          '{"range":{"from":"1602637140000","to":"1602637261000"},"datasource":"pyroscopeUid","queries":[{"labelSelector":"{ip=\\"192.168.0.1\\"}","groupBy":[],"profileTypeId":"","queryType":"profile","spanSelector":["6605c7b08e715d6c"],"refId":""}]}'
+        )}`
+      );
+    });
+
+    it('from tags and process tags as well', () => {
+      const createLink = setupSpanLinkFactory(
+        {
+          tags: [{ key: 'ip' }, { key: 'host' }],
+        },
+        '',
+        dummyDataFrameForProfiles
+      );
+      expect(createLink).toBeDefined();
+      const links = createLink!(
+        createTraceSpan({
+          process: {
+            serviceName: 'service',
+            tags: [
+              { key: 'hostname', value: 'hostname1' },
+              { key: 'ip', value: '192.168.0.1' },
+            ],
+          },
+        })
+      );
+      const linkDef = links?.[0];
+      expect(linkDef).toBeDefined();
+      expect(linkDef?.type).toBe(SpanLinkType.Profiles);
+      expect(linkDef!.href).toBe(
+        `/explore?left=${encodeURIComponent(
+          '{"range":{"from":"1602637140000","to":"1602637261000"},"datasource":"pyroscopeUid","queries":[{"labelSelector":"{ip=\\"192.168.0.1\\", host=\\"host\\"}","groupBy":[],"profileTypeId":"","queryType":"profile","spanSelector":["6605c7b08e715d6c"],"refId":""}]}'
+        )}`
+      );
+    });
+
+    it('creates link from dataFrame', () => {
+      const splitOpenFn = jest.fn();
+      const createLink = createSpanLinkFactory({
+        splitOpenFn,
+        dataFrame: createDataFrame({
+          fields: [
+            { name: 'traceID', values: ['testTraceId'] },
+            {
+              name: 'spanID',
+              config: { links: [{ title: 'link', url: '${__data.fields.spanID}' }] },
+              values: ['testSpanId'],
+            },
+          ],
+        }),
+        trace: dummyTraceData,
+      });
+      expect(createLink).toBeDefined();
+      const links = createLink!(createTraceSpan());
+
+      const linkDef = links?.[0];
+      expect(linkDef).toBeDefined();
+      expect(linkDef?.type).toBe(SpanLinkType.Unknown);
+      expect(linkDef!.href).toBe('testSpanId');
+    });
+
+    it('handles renamed tags', () => {
+      const createLink = setupSpanLinkFactory(
+        {
+          tags: [
+            { key: 'service.name', value: 'service' },
+            { key: 'k8s.pod.name', value: 'pod' },
+          ],
+        },
+        '',
+        dummyDataFrameForProfiles
+      );
+      expect(createLink).toBeDefined();
+      const links = createLink!(
+        createTraceSpan({
+          process: {
+            serviceName: 'service',
+            tags: [
+              { key: 'service.name', value: 'serviceName' },
+              { key: 'k8s.pod.name', value: 'podName' },
+            ],
+          },
+        })
+      );
+
+      const linkDef = links?.[0];
+      expect(linkDef).toBeDefined();
+      expect(linkDef?.type).toBe(SpanLinkType.Profiles);
+      expect(linkDef!.href).toBe(
+        `/explore?left=${encodeURIComponent(
+          '{"range":{"from":"1602637140000","to":"1602637261000"},"datasource":"pyroscopeUid","queries":[{"labelSelector":"{service=\\"serviceName\\", pod=\\"podName\\"}","groupBy":[],"profileTypeId":"","queryType":"profile","spanSelector":["6605c7b08e715d6c"],"refId":""}]}'
+        )}`
+      );
+    });
+
+    it('handles incomplete renamed tags', () => {
+      const createLink = setupSpanLinkFactory(
+        {
+          tags: [
+            { key: 'service.name', value: '' },
+            { key: 'k8s.pod.name', value: 'pod' },
+          ],
+        },
+        '',
+        dummyDataFrameForProfiles
+      );
+      expect(createLink).toBeDefined();
+      const links = createLink!(
+        createTraceSpan({
+          process: {
+            serviceName: 'service',
+            tags: [
+              { key: 'service.name', value: 'serviceName' },
+              { key: 'k8s.pod.name', value: 'podName' },
+            ],
+          },
+        })
+      );
+
+      const linkDef = links?.[0];
+      expect(linkDef).toBeDefined();
+      expect(linkDef?.type).toBe(SpanLinkType.Profiles);
+      expect(linkDef!.href).toBe(
+        `/explore?left=${encodeURIComponent(
+          '{"range":{"from":"1602637140000","to":"1602637261000"},"datasource":"pyroscopeUid","queries":[{"labelSelector":"{service.name=\\"serviceName\\", pod=\\"podName\\"}","groupBy":[],"profileTypeId":"","queryType":"profile","spanSelector":["6605c7b08e715d6c"],"refId":""}]}'
+        )}`
+      );
+    });
+
+    it('interpolates span intrinsics', () => {
+      const createLink = setupSpanLinkFactory(
+        {
+          tags: [{ key: 'name', value: 'spanName' }],
+        },
+        '',
+        dummyDataFrameForProfiles
+      );
+      expect(createLink).toBeDefined();
+      const links = createLink!(createTraceSpan());
+      expect(links).toBeDefined();
+      expect(links![0].type).toBe(SpanLinkType.Profiles);
+      expect(decodeURIComponent(links![0].href)).toContain('spanName=\\"operation\\"');
+    });
+  });
 });
 
 describe('dataFrame links', () => {
@@ -1272,7 +1509,11 @@ describe('dataFrame links', () => {
   });
 });
 
-function setupSpanLinkFactory(options: Partial<TraceToLogsOptionsV2> = {}, datasourceUid = 'lokiUid') {
+function setupSpanLinkFactory(
+  options: Partial<TraceToLogsOptionsV2> = {},
+  datasourceUid = 'lokiUid',
+  dummyDataFrameForProfiles?: DataFrame
+) {
   const splitOpenFn = jest.fn();
   return createSpanLinkFactory({
     splitOpenFn,
@@ -1281,13 +1522,18 @@ function setupSpanLinkFactory(options: Partial<TraceToLogsOptionsV2> = {}, datas
       datasourceUid,
       ...options,
     },
+    traceToProfilesOptions: {
+      customQuery: false,
+      datasourceUid: 'pyroscopeUid',
+      ...options,
+    },
     createFocusSpanLink: (traceId, spanId) => {
       return {
         href: `${traceId}-${spanId}`,
       } as unknown as LinkModel;
     },
     trace: dummyTraceData,
-    dataFrame: dummyDataFrame,
+    dataFrame: dummyDataFrameForProfiles ? dummyDataFrameForProfiles : dummyDataFrame,
   });
 }
 
@@ -1308,6 +1554,10 @@ function createTraceSpan(overrides: Partial<TraceSpan> = {}) {
         key: 'host',
         value: 'host',
       },
+      {
+        key: 'pyroscope.profiling.enabled',
+        value: 'hdgfljn23u982nj',
+      },
     ],
     process: {
       serviceName: 'test service',

public/app/features/explore/TraceView/createSpanLink.tsx

@@ -19,6 +19,7 @@ import { DataQuery } from '@grafana/schema';
 import { Icon } from '@grafana/ui';
 import { TraceToLogsOptionsV2, TraceToLogsTag } from 'app/core/components/TraceToLogs/TraceToLogsSettings';
 import { TraceToMetricQuery, TraceToMetricsOptions } from 'app/core/components/TraceToMetrics/TraceToMetricsSettings';
+import { TraceToProfilesOptions } from 'app/core/components/TraceToProfiles/TraceToProfilesSettings';
 import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
 import { PromQuery } from 'app/plugins/datasource/prometheus/types';
 
@@ -37,6 +38,7 @@ export function createSpanLinkFactory({
   splitOpenFn,
   traceToLogsOptions,
   traceToMetricsOptions,
+  traceToProfilesOptions,
   dataFrame,
   createFocusSpanLink,
   trace,
@@ -44,6 +46,7 @@ export function createSpanLinkFactory({
   splitOpenFn: SplitOpen;
   traceToLogsOptions?: TraceToLogsOptionsV2;
   traceToMetricsOptions?: TraceToMetricsOptions;
+  traceToProfilesOptions?: TraceToProfilesOptions;
   dataFrame?: DataFrame;
   createFocusSpanLink?: (traceId: string, spanId: string) => LinkModel<Field>;
   trace: Trace;
@@ -72,17 +75,26 @@ export function createSpanLinkFactory({
       scopedVars = {
         ...scopedVars,
         ...scopedVarsFromSpan(span),
+        ...scopedVarsFromTags(span, traceToProfilesOptions),
       };
       // We should be here only if there are some links in the dataframe
       const fields = dataFrame.fields.filter((f) => Boolean(f.config.links?.length))!;
       try {
+        let profilesDataSourceSettings: DataSourceInstanceSettings<DataSourceJsonData> | undefined;
+        if (traceToProfilesOptions?.datasourceUid) {
+          profilesDataSourceSettings = getDatasourceSrv().getInstanceSettings(traceToProfilesOptions.datasourceUid);
+        }
+        const hasConfiguredPyroscopeDS = profilesDataSourceSettings?.type === 'grafana-pyroscope-datasource';
+        const hasPyroscopeProfile = span.tags.filter((tag) => tag.key === 'pyroscope.profiling.enabled').length > 0;
+        const shouldCreatePyroscopeLink = hasConfiguredPyroscopeDS && hasPyroscopeProfile;
+
         let links: ExploreFieldLinkModel[] = [];
         fields.forEach((field) => {
           const fieldLinksForExplore = getFieldLinksForExplore({
             field,
             rowIndex: span.dataFrameRowIndex!,
             splitOpenFn,
-            range: getTimeRangeFromSpan(span),
+            range: getTimeRangeFromSpan(span, undefined, undefined, shouldCreatePyroscopeLink),
             dataFrame,
             vars: scopedVars,
           });
@@ -96,7 +108,7 @@ export function createSpanLinkFactory({
             onClick: link.onClick,
             content: <Icon name="link" title={link.title || 'Link'} />,
             field: link.origin,
-            type: SpanLinkType.Unknown,
+            type: shouldCreatePyroscopeLink ? SpanLinkType.Profiles : SpanLinkType.Unknown,
           };
         });
 
@@ -115,10 +127,14 @@ export function createSpanLinkFactory({
 /**
  * Default keys to use when there are no configured tags.
  */
-const defaultKeys = ['cluster', 'hostname', 'namespace', 'pod', 'service.name', 'service.namespace'].map((k) => ({
+const formatDefaultKeys = (keys: string[]) => {
-  key: k,
+  return keys.map((k) => ({
-  value: k.includes('.') ? k.replace('.', '_') : undefined,
+    key: k,
-}));
+    value: k.includes('.') ? k.replace('.', '_') : undefined,
+  }));
+};
+const defaultKeys = formatDefaultKeys(['cluster', 'hostname', 'namespace', 'pod', 'service.name', 'service.namespace']);
+const defaultProfilingKeys = formatDefaultKeys(['service.name', 'service.namespace']);
 
 function legacyCreateSpanLinkFactory(
   splitOpenFn: SplitOpen,
@@ -514,16 +530,19 @@ function getFormattedTags(
 function getTimeRangeFromSpan(
   span: TraceSpan,
   timeShift: { startMs: number; endMs: number } = { startMs: 0, endMs: 0 },
-  isSplunkDS = false
+  isSplunkDS = false,
+  shouldCreatePyroscopeLink = false
 ): TimeRange {
-  const adjustedStartTime = Math.floor(span.startTime / 1000 + timeShift.startMs);
+  let adjustedStartTime = Math.floor(span.startTime / 1000 + timeShift.startMs);
-  const from = dateTime(adjustedStartTime);
   const spanEndMs = (span.startTime + span.duration) / 1000;
   let adjustedEndTime = Math.floor(spanEndMs + timeShift.endMs);
 
   // Splunk requires a time interval of >= 1s, rather than >=1ms like Loki timerange in below elseif block
   if (isSplunkDS && adjustedEndTime - adjustedStartTime < 1000) {
     adjustedEndTime = adjustedStartTime + 1000;
+  } else if (shouldCreatePyroscopeLink) {
+    adjustedStartTime = adjustedStartTime - 60000;
+    adjustedEndTime = adjustedEndTime + 60000;
   } else if (adjustedStartTime === adjustedEndTime) {
     // Because we can only pass milliseconds in the url we need to check if they equal.
     // We need end time to be later than start time
@@ -531,6 +550,7 @@ function getTimeRangeFromSpan(
   }
 
   const to = dateTime(adjustedEndTime);
+  const from = dateTime(adjustedStartTime);
 
   // Beware that public/app/features/explore/state/main.ts SplitOpen fn uses the range from here. No matter what is in the url.
   return {
@@ -617,3 +637,27 @@ function scopedVarsFromSpan(span: TraceSpan): ScopedVars {
     },
   };
 }
+
+/**
+ * Variables from tags that can be used in the query
+ * @param span
+ */
+function scopedVarsFromTags(span: TraceSpan, traceToProfilesOptions: TraceToProfilesOptions | undefined): ScopedVars {
+  let tags: ScopedVars = {};
+
+  if (traceToProfilesOptions) {
+    const profileTags =
+      traceToProfilesOptions.tags && traceToProfilesOptions.tags.length > 0
+        ? traceToProfilesOptions.tags
+        : defaultProfilingKeys;
+
+    tags = {
+      __tags: {
+        text: 'Tags',
+        value: getFormattedTags(span, profileTags),
+      },
+    };
+  }
+
+  return tags;
+}

public/app/plugins/datasource/grafana-pyroscope-datasource/QueryEditor/ProfileTypesCascader.tsx

@@ -10,6 +10,7 @@ type Props = {
   profileTypes?: ProfileTypeMessage[];
   onChange: (value: string) => void;
   placeholder?: string;
+  width?: number;
 };
 
 export function ProfileTypesCascader(props: Props) {
@@ -25,6 +26,7 @@ export function ProfileTypesCascader(props: Props) {
       onSelect={props.onChange}
       options={cascaderOptions}
       changeOnSelect={false}
+      width={props.width ?? 26}
     />
   );
 }

public/app/plugins/datasource/grafana-pyroscope-datasource/QueryEditor/QueryOptions.tsx

@@ -2,6 +2,7 @@ import { css } from '@emotion/css';
 import React from 'react';
 
 import { CoreApp, GrafanaTheme2, SelectableValue } from '@grafana/data';
+import { config } from '@grafana/runtime';
 import { useStyles2, RadioButtonGroup, MultiSelect, Input } from '@grafana/ui';
 
 import { QueryOptionGroup } from '../../prometheus/querybuilder/shared/QueryOptionGroup';
@@ -83,6 +84,21 @@ export function QueryOptions({ query, onQueryChange, app, labels }: Props) {
               }}
             />
           </EditorField>
+          {config.featureToggles.traceToProfiles && (
+            <EditorField label={'Span ID'} tooltip={<>Sets the span ID from which to search for profiles.</>}>
+              <Input
+                value={query.spanSelector || ['']}
+                type="string"
+                placeholder="64f170a95f537095"
+                onChange={(event: React.SyntheticEvent<HTMLInputElement>) => {
+                  onQueryChange({
+                    ...query,
+                    spanSelector: event.currentTarget.value !== '' ? [event.currentTarget.value] : [],
+                  });
+                }}
+              />
+            </EditorField>
+          )}
           <EditorField label={'Max Nodes'} tooltip={<>Sets the maximum number of nodes to return in the flamegraph.</>}>
             <Input
               value={query.maxNodes || ''}

public/app/plugins/datasource/grafana-pyroscope-datasource/dataquery.cue

@@ -31,6 +31,8 @@ composableKinds: DataQuery: {
 
 				// Specifies the query label selectors.
 				labelSelector: string | *"{}"
+				// Specifies the query span selectors.
+				spanSelector?: [...string]
 				// Specifies the type of profile to query.
 				profileTypeId: string
 				// Allows to group the results.

public/app/plugins/datasource/grafana-pyroscope-datasource/dataquery.gen.ts

@@ -31,9 +31,14 @@ export interface GrafanaPyroscope extends common.DataQuery {
    * Specifies the type of profile to query.
    */
   profileTypeId: string;
+  /**
+   * Specifies the query span selectors.
+   */
+  spanSelector?: Array<string>;
 }
 
 export const defaultGrafanaPyroscope: Partial<GrafanaPyroscope> = {
   groupBy: [],
   labelSelector: '{}',
+  spanSelector: [],
 };

public/app/plugins/datasource/tempo/configuration/ConfigEditor.tsx

@@ -18,6 +18,7 @@ import { Divider } from 'app/core/components/Divider';
 import { NodeGraphSection } from 'app/core/components/NodeGraphSettings';
 import { TraceToLogsSection } from 'app/core/components/TraceToLogs/TraceToLogsSettings';
 import { TraceToMetricsSection } from 'app/core/components/TraceToMetrics/TraceToMetricsSettings';
+import { TraceToProfilesSection } from 'app/core/components/TraceToProfiles/TraceToProfilesSettings';
 import { SpanBarSection } from 'app/features/explore/TraceView/components/settings/SpanBarSettings';
 
 import { LokiSearchSettings } from './LokiSearchSettings';
@@ -60,6 +61,13 @@ export const ConfigEditor = ({ options, onOptionsChange }: Props) => {
         </>
       ) : null}
 
+      {config.featureToggles.traceToProfiles && (
+        <>
+          <TraceToProfilesSection options={options} onOptionsChange={onOptionsChange} />
+          <Divider />
+        </>
+      )}
+
       <ConfigSection
         title="Additional settings"
         description="Additional settings are optional settings that can be configured for more control over your data source."

public/app/plugins/datasource/tempo/datasource.ts

@@ -631,7 +631,7 @@ export class TempoDatasource extends DataSourceWithBackend<TempoQuery, TempoJson
         if (response.error) {
           return response;
         }
-        return transformTrace(response, this.nodeGraph?.enabled);
+        return transformTrace(response, this.instanceSettings, this.nodeGraph?.enabled);
       })
     );
   }

public/app/plugins/datasource/tempo/resultTransformer.ts

@@ -18,11 +18,18 @@ import {
   createTheme,
   DataFrameDTO,
   toDataFrame,
+  DataLink,
+  DataSourceJsonData,
+  Field,
+  DataLinkConfigOrigin,
 } from '@grafana/data';
+import { config } from '@grafana/runtime';
+import { TraceToProfilesData } from 'app/core/components/TraceToProfiles/TraceToProfilesSettings';
+import { getDatasourceSrv } from 'app/features/plugins/datasource_srv';
 
 import { SearchTableType } from './dataquery.gen';
 import { createGraphFrames } from './graphTransform';
-import { Span, SpanAttributes, Spanset, TraceSearchMetadata } from './types';
+import { Span, SpanAttributes, Spanset, TempoJsonData, TraceSearchMetadata } from './types';
 
 export function createTableFrame(
   logsFrame: DataFrame | DataFrameDTO,
@@ -491,13 +498,56 @@ function getOTLPReferences(
   return links;
 }
 
-export function transformTrace(response: DataQueryResponse, nodeGraph = false): DataQueryResponse {
+export const RelatedProfilesTitle = 'Related profiles';
+
+export function transformTrace(
+  response: DataQueryResponse,
+  instanceSettings: DataSourceInstanceSettings<TempoJsonData>,
+  nodeGraph = false
+): DataQueryResponse {
   const frame = response.data[0];
 
   if (!frame) {
     return emptyDataQueryResponse;
   }
 
+  // Get profiles links
+  if (config.featureToggles.traceToProfiles) {
+    const traceToProfilesData: TraceToProfilesData | undefined = instanceSettings?.jsonData;
+    const traceToProfilesOptions = traceToProfilesData?.tracesToProfiles;
+    let profilesDataSourceSettings: DataSourceInstanceSettings<DataSourceJsonData> | undefined;
+    if (traceToProfilesOptions?.datasourceUid) {
+      profilesDataSourceSettings = getDatasourceSrv().getInstanceSettings(traceToProfilesOptions.datasourceUid);
+    }
+
+    if (traceToProfilesOptions && profilesDataSourceSettings) {
+      const customQuery = traceToProfilesOptions.customQuery ? traceToProfilesOptions.query : undefined;
+      const dataLink: DataLink = {
+        title: RelatedProfilesTitle,
+        url: '',
+        internal: {
+          datasourceUid: profilesDataSourceSettings.uid,
+          datasourceName: profilesDataSourceSettings.name,
+          query: {
+            labelSelector: customQuery ? customQuery : '{${__tags}}',
+            groupBy: [],
+            profileTypeId: traceToProfilesOptions.profileTypeId ?? '',
+            queryType: 'profile',
+            spanSelector: ['${__span.spanId}'],
+            refId: 'profile',
+          },
+        },
+        origin: DataLinkConfigOrigin.Datasource,
+      };
+
+      frame.fields.forEach((field: Field) => {
+        if (field.name === 'tags') {
+          field.config.links = [dataLink];
+        }
+      });
+    }
+  }
+
   let data = [...response.data];
   if (nodeGraph) {
     data.push(...createGraphFrames(toDataFrame(frame)));