Plugins: Support for distributed tracing in backend plugins SDK (#63714)

* Tracing: Pass OTLP address and propagation format to plugins

* Fix unit tests

* Fix indentation

* Fix plugin manager integration tests

* Goimports

* Pass plugin version to plugins

* Do not add GF_PLUGIN_VERSION if plugin version is not set, add tests

* Allow disabling plugins distributed tracing on a per-plugin basis

* Moved disabled plugins to tracing.opentelemetry config section

* Pre-allocate DisabledPlugins map to the correct size

* Moved disable tracing setting flags in plugin settings

* Renamed plugin env vars for tracing endpoint and propagation

* Fix plugin initializer tests

* Refactoring: Moved OpentelemetryCfg from pkg/infra to pkg/plugins

* Changed GetSection to Section in parseSettingsOpentelemetry

* Add tests for NewOpentelemetryCfg

* Fix test case names in TestNewOpentelemetryCfg

* OpenTelemetry: Remove redundant error checks

pkg/api/plugin_resource_test.go

@@ -50,6 +50,7 @@ func TestCallResource(t *testing.T) {
 	require.NoError(t, err)
 
 	cfg := setting.NewCfg()
+
 	cfg.StaticRootPath = staticRootPath
 	cfg.IsFeatureToggleEnabled = func(_ string) bool {
 		return false
@@ -58,7 +59,9 @@ func TestCallResource(t *testing.T) {
 
 	coreRegistry := coreplugin.ProvideCoreRegistry(nil, &cloudwatch.CloudWatchService{}, nil, nil, nil, nil,
 		nil, nil, nil, nil, testdatasource.ProvideService(cfg, featuremgmt.WithFeatures()), nil, nil, nil, nil, nil, nil)
-	pCfg := config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
+	var pCfg *config.Cfg
+	pCfg, err = config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
+	require.NoError(t, err)
 	reg := registry.ProvideService()
 	cdn := pluginscdn.ProvideService(pCfg)
 	l := loader.ProvideService(pCfg, fakes.NewFakeLicensingService(), signature.NewUnsignedAuthorizer(pCfg),

pkg/infra/tracing/opentelemetry_tracing.go

@@ -36,10 +36,11 @@ const (
 )
 
 type Opentelemetry struct {
-	enabled       string
+	Enabled       string
-	address       string
+	Address       string
-	propagation   string
+	Propagation   string
 	customAttribs []attribute.KeyValue
+
 	log log.Logger
 
 	tracerProvider tracerProvider
@@ -78,39 +79,29 @@ func (noopTracerProvider) Shutdown(ctx context.Context) error {
 }
 
 func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
-	section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry")
+	section := ots.Cfg.Raw.Section("tracing.opentelemetry")
-	if err != nil {
+	var err error
-		return err
-	}
-
 	ots.customAttribs, err = splitCustomAttribs(section.Key("custom_attributes").MustString(""))
 	if err != nil {
 		return err
 	}
 
-	section, err = ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
+	section = ots.Cfg.Raw.Section("tracing.opentelemetry.jaeger")
-	if err != nil {
+	ots.Enabled = noopExporter
-		return err
-	}
-	ots.enabled = noopExporter
 
-	ots.address = section.Key("address").MustString("")
+	ots.Address = section.Key("address").MustString("")
-	ots.propagation = section.Key("propagation").MustString("")
+	ots.Propagation = section.Key("propagation").MustString("")
-	if ots.address != "" {
+	if ots.Address != "" {
-		ots.enabled = jaegerExporter
+		ots.Enabled = jaegerExporter
 		return nil
 	}
 
-	section, err = ots.Cfg.Raw.GetSection("tracing.opentelemetry.otlp")
+	section = ots.Cfg.Raw.Section("tracing.opentelemetry.otlp")
-	if err != nil {
+	ots.Address = section.Key("address").MustString("")
-		return err
+	if ots.Address != "" {
+		ots.Enabled = otlpExporter
 	}
-
+	ots.Propagation = section.Key("propagation").MustString("")
-	ots.address = section.Key("address").MustString("")
-	if ots.address != "" {
-		ots.enabled = otlpExporter
-	}
-	ots.propagation = section.Key("propagation").MustString("")
 	return nil
 }
 
@@ -132,7 +123,7 @@ func splitCustomAttribs(s string) ([]attribute.KeyValue, error) {
 
 func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
 	// Create the Jaeger exporter
-	exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
+	exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.Address)))
 	if err != nil {
 		return nil, err
 	}
@@ -160,7 +151,7 @@ func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider,
 }
 
 func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
-	client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.address), otlptracegrpc.WithInsecure())
+	client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.Address), otlptracegrpc.WithInsecure())
 	exp, err := otlptrace.New(context.Background(), client)
 	if err != nil {
 		return nil, err
@@ -201,7 +192,7 @@ func (ots *Opentelemetry) initNoopTracerProvider() (tracerProvider, error) {
 func (ots *Opentelemetry) initOpentelemetryTracer() error {
 	var tp tracerProvider
 	var err error
-	switch ots.enabled {
+	switch ots.Enabled {
 	case jaegerExporter:
 		tp, err = ots.initJaegerTracerProvider()
 		if err != nil {
@@ -222,12 +213,12 @@ func (ots *Opentelemetry) initOpentelemetryTracer() error {
 	// Register our TracerProvider as the global so any imported
 	// instrumentation in the future will default to using it
 	// only if tracing is enabled
-	if ots.enabled != "" {
+	if ots.Enabled != "" {
 		otel.SetTracerProvider(tp)
 	}
 
 	propagators := []propagation.TextMapPropagator{}
-	for _, p := range strings.Split(ots.propagation, ",") {
+	for _, p := range strings.Split(ots.Propagation, ",") {
 		switch p {
 		case w3cPropagator:
 			propagators = append(propagators, propagation.TraceContext{}, propagation.Baggage{})

pkg/infra/tracing/optentelemetry_tracing_test.go

@@ -62,7 +62,7 @@ func TestOptentelemetry_ParseSettingsOpentelemetry(t *testing.T) {
 	otlpsect := cfg.Raw.Section("tracing.opentelemetry.otlp")
 
 	assert.NoError(t, otel.parseSettingsOpentelemetry())
-	assert.Equal(t, noopExporter, otel.enabled)
+	assert.Equal(t, noopExporter, otel.Enabled)
 
 	otelsect.Key("custom_attributes")
 	assert.NoError(t, otel.parseSettingsOpentelemetry())
@@ -78,12 +78,12 @@ func TestOptentelemetry_ParseSettingsOpentelemetry(t *testing.T) {
 
 	jaegersect.Key("address").SetValue("somehost:6831")
 	assert.NoError(t, otel.parseSettingsOpentelemetry())
-	assert.Equal(t, "somehost:6831", otel.address)
+	assert.Equal(t, "somehost:6831", otel.Address)
-	assert.Equal(t, jaegerExporter, otel.enabled)
+	assert.Equal(t, jaegerExporter, otel.Enabled)
 
 	jaegersect.Key("address").SetValue("")
 	otlpsect.Key("address").SetValue("somehost:4317")
 	assert.NoError(t, otel.parseSettingsOpentelemetry())
-	assert.Equal(t, "somehost:4317", otel.address)
+	assert.Equal(t, "somehost:4317", otel.Address)
-	assert.Equal(t, otlpExporter, otel.enabled)
+	assert.Equal(t, otlpExporter, otel.Enabled)
 }

pkg/infra/tracing/test_helper.go

@@ -16,7 +16,7 @@ func InitializeTracerForTest() Tracer {
 	tp, _ := initTracerProvider(exp)
 	otel.SetTracerProvider(tp)
 
-	ots := &Opentelemetry{propagation: "jaeger,w3c", tracerProvider: tp}
+	ots := &Opentelemetry{Propagation: "jaeger,w3c", tracerProvider: tp}
 	_ = ots.initOpentelemetryTracer()
 	return ots
 }

pkg/infra/tracing/tracing.go

@@ -100,25 +100,35 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) {
 }
 
 func parseSettings(cfg *setting.Cfg) (*Opentracing, *Opentelemetry, error) {
+	ts, err := parseSettingsOpentracing(cfg)
+	if err != nil {
+		return ts, nil, err
+	}
+	ots, err := ParseSettingsOpentelemetry(cfg)
+	return ts, ots, err
+}
+
+func parseSettingsOpentracing(cfg *setting.Cfg) (*Opentracing, error) {
 	ts := &Opentracing{
 		Cfg: cfg,
 		log: log.New("tracing"),
 	}
-	err := ts.parseSettings()
+	if err := ts.parseSettings(); err != nil {
-	if err != nil {
+		return ts, err
-		return ts, nil, err
 	}
 	if ts.enabled {
 		cfg.Logger.Warn("[Deprecated] the configuration setting 'tracing.jaeger' is deprecated, please use 'tracing.opentelemetry.jaeger' instead")
-		return ts, nil, nil
+	}
+	return ts, nil
 }
 
+func ParseSettingsOpentelemetry(cfg *setting.Cfg) (*Opentelemetry, error) {
 	ots := &Opentelemetry{
 		Cfg: cfg,
 		log: log.New("tracing"),
 	}
-	err = ots.parseSettingsOpentelemetry()
+	err := ots.parseSettingsOpentelemetry()
-	return ts, ots, err
+	return ots, err
 }
 
 type traceKey struct{}

pkg/plugins/config/config.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"fmt"
 	"strings"
 
 	"github.com/grafana/grafana-azure-sdk-go/azsettings"
@@ -31,13 +32,15 @@ type Cfg struct {
 	LogDatasourceRequests bool
 
 	PluginsCDNURLTemplate string
+
+	Opentelemetry OpentelemetryCfg
 }
 
-func ProvideConfig(settingProvider setting.Provider, grafanaCfg *setting.Cfg) *Cfg {
+func ProvideConfig(settingProvider setting.Provider, grafanaCfg *setting.Cfg) (*Cfg, error) {
 	return NewCfg(settingProvider, grafanaCfg)
 }
 
-func NewCfg(settingProvider setting.Provider, grafanaCfg *setting.Cfg) *Cfg {
+func NewCfg(settingProvider setting.Provider, grafanaCfg *setting.Cfg) (*Cfg, error) {
 	logger := log.New("plugin.cfg")
 
 	aws := settingProvider.Section("aws")
@@ -53,6 +56,10 @@ func NewCfg(settingProvider setting.Provider, grafanaCfg *setting.Cfg) *Cfg {
 		allowedUnsigned = strings.Split(settingProvider.KeyValue("plugins", "allow_loading_unsigned_plugins").Value(), ",")
 	}
 
+	otelCfg, err := NewOpentelemetryCfg(grafanaCfg)
+	if err != nil {
+		return nil, fmt.Errorf("new opentelemetry cfg: %w", err)
+	}
 	return &Cfg{
 		log:                     logger,
 		PluginsPath:             grafanaCfg.PluginsPath,
@@ -65,7 +72,8 @@ func NewCfg(settingProvider setting.Provider, grafanaCfg *setting.Cfg) *Cfg {
 		Azure:                   grafanaCfg.Azure,
 		LogDatasourceRequests:   grafanaCfg.PluginLogBackendRequests,
 		PluginsCDNURLTemplate:   grafanaCfg.PluginsCDNURLTemplate,
-	}
+		Opentelemetry:           otelCfg,
+	}, nil
 }
 
 func extractPluginSettings(settingProvider setting.Provider) setting.PluginSettings {

pkg/plugins/config/tracing.go

@@ -0,0 +1,38 @@
+package config
+
+import (
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/infra/tracing"
+	"github.com/grafana/grafana/pkg/setting"
+)
+
+const otlpExporter string = "otlp"
+
+// OpentelemetryCfg contains the Opentelemetry address and propagation config values.
+// This is used to export the Opentelemetry (OTLP) config without exposing the whole *setting.Cfg.
+type OpentelemetryCfg struct {
+	Address     string
+	Propagation string
+}
+
+// IsEnabled returns true if OTLP tracing is enabled (address set)
+func (c OpentelemetryCfg) IsEnabled() bool {
+	return c.Address != ""
+}
+
+// NewOpentelemetryCfg creates a new OpentelemetryCfg based on the provided Grafana config.
+// If Opentelemetry (OTLP) is disabled, a zero-value OpentelemetryCfg is returned.
+func NewOpentelemetryCfg(grafanaCfg *setting.Cfg) (OpentelemetryCfg, error) {
+	ots, err := tracing.ParseSettingsOpentelemetry(grafanaCfg)
+	if err != nil {
+		return OpentelemetryCfg{}, fmt.Errorf("parse settings: %w", err)
+	}
+	if ots.Enabled != otlpExporter {
+		return OpentelemetryCfg{}, nil
+	}
+	return OpentelemetryCfg{
+		Address:     ots.Address,
+		Propagation: ots.Propagation,
+	}, nil
+}

pkg/plugins/config/tracing_test.go

@@ -0,0 +1,50 @@
+package config
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewOpentelemetryCfg(t *testing.T) {
+	t.Run("empty", func(t *testing.T) {
+		cfg := setting.NewCfg()
+
+		otelCfg, err := NewOpentelemetryCfg(cfg)
+		require.NoError(t, err)
+		assert.False(t, otelCfg.IsEnabled(), "otel should be disabled")
+		assert.Empty(t, otelCfg.Address)
+		assert.Empty(t, otelCfg.Propagation)
+	})
+
+	t.Run("enabled", func(t *testing.T) {
+		for _, tc := range []struct {
+			name        string
+			propagation string
+		}{
+			{"empty", ""},
+			{"jaeger", "jaeger"},
+			{"w3c", "w3c"},
+			{"multiple", "jaeger,w3c"},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				const address = "127.0.0.1:4317"
+
+				cfg := setting.NewCfg()
+				otlpSect := cfg.Raw.Section("tracing.opentelemetry.otlp")
+				otlpSect.Key("address").SetValue(address)
+				if tc.propagation != "" {
+					otlpSect.Key("propagation").SetValue(tc.propagation)
+				}
+
+				otelCfg, err := NewOpentelemetryCfg(cfg)
+				require.NoError(t, err)
+				assert.True(t, otelCfg.IsEnabled(), "otel should be enabled")
+				assert.Equal(t, address, otelCfg.Address)
+				assert.Equal(t, tc.propagation, otelCfg.Propagation)
+			})
+		}
+	})
+}

pkg/plugins/manager/loader/initializer/initializer.go

@@ -51,6 +51,9 @@ func (i *Initializer) envVars(plugin *plugins.Plugin) []string {
 	hostEnv := []string{
 		fmt.Sprintf("GF_VERSION=%s", i.cfg.BuildVersion),
 	}
+	if plugin.Info.Version != "" {
+		hostEnv = append(hostEnv, fmt.Sprintf("GF_PLUGIN_VERSION=%s", plugin.Info.Version))
+	}
 
 	if i.license != nil {
 		hostEnv = append(
@@ -64,7 +67,22 @@ func (i *Initializer) envVars(plugin *plugins.Plugin) []string {
 
 	hostEnv = append(hostEnv, i.awsEnvVars()...)
 	hostEnv = append(hostEnv, azsettings.WriteToEnvStr(i.cfg.Azure)...)
-	return getPluginSettings(plugin.ID, i.cfg).asEnvVar("GF_PLUGIN", hostEnv)
+
+	// Tracing
+	pluginTracingEnabled := true
+	if v, exists := i.cfg.PluginSettings[plugin.ID]["tracing"]; exists {
+		pluginTracingEnabled = v != "false"
+	}
+	if i.cfg.Opentelemetry.IsEnabled() && pluginTracingEnabled {
+		hostEnv = append(
+			hostEnv,
+			fmt.Sprintf("GF_INSTANCE_OTLP_ADDRESS=%s", i.cfg.Opentelemetry.Address),
+			fmt.Sprintf("GF_INSTANCE_OTLP_PROPAGATION=%s", i.cfg.Opentelemetry.Propagation),
+		)
+	}
+
+	ev := getPluginSettings(plugin.ID, i.cfg).asEnvVar("GF_PLUGIN", hostEnv)
+	return ev
 }
 
 func (i *Initializer) awsEnvVars() []string {

pkg/plugins/manager/loader/initializer/initializer_test.go

@@ -2,9 +2,12 @@ package initializer
 
 import (
 	"context"
+	"strings"
 	"testing"
 
+	"github.com/grafana/grafana/pkg/setting"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/backendplugin"
@@ -129,6 +132,53 @@ func TestInitializer_Initialize(t *testing.T) {
 }
 
 func TestInitializer_envVars(t *testing.T) {
+	t.Run("version", func(t *testing.T) {
+		for _, tc := range []struct {
+			name  string
+			setup func(p *plugins.Plugin)
+			exp   func(t *testing.T, i *Initializer, p *plugins.Plugin)
+		}{
+			{
+				name: "not provided",
+				setup: func(p *plugins.Plugin) {
+					p.Info = plugins.Info{}
+				},
+				exp: func(t *testing.T, i *Initializer, p *plugins.Plugin) {
+					for _, k := range i.envVars(p) {
+						if strings.HasPrefix("GF_PLUGIN_VERSION=", k) {
+							require.Fail(t, "found unexpected env var GF_PLUGIN_VERSION")
+						}
+					}
+				},
+			},
+			{
+				name: "provided",
+				setup: func(p *plugins.Plugin) {
+					p.Info = plugins.Info{Version: "0.1"}
+				},
+				exp: func(t *testing.T, i *Initializer, p *plugins.Plugin) {
+					require.Contains(t, i.envVars(p), "GF_PLUGIN_VERSION=0.1")
+				},
+			},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				p := &plugins.Plugin{
+					JSONData: plugins.JSONData{
+						ID:   "test",
+						Info: plugins.Info{Version: "0.1"},
+					},
+				}
+				tc.setup(p)
+				i := &Initializer{
+					cfg:             &config.Cfg{},
+					log:             log.NewTestLogger(),
+					backendProvider: &fakeBackendProvider{plugin: p},
+				}
+				tc.exp(t, i, p)
+			})
+		}
+	})
+
 	t.Run("backend datasource with license", func(t *testing.T) {
 		p := &plugins.Plugin{
 			JSONData: plugins.JSONData{
@@ -169,6 +219,114 @@ func TestInitializer_envVars(t *testing.T) {
 	})
 }
 
+func TestInitializer_tracingEnvironmentVariables(t *testing.T) {
+	const pluginID = "plugin_id"
+
+	p := &plugins.Plugin{
+		JSONData: plugins.JSONData{ID: pluginID},
+	}
+
+	defaultOtelCfg := config.OpentelemetryCfg{
+		Address:     "127.0.0.1:4317",
+		Propagation: "",
+	}
+
+	expDefaultOtlp := func(t *testing.T, envVars []string) {
+		found := map[string]bool{
+			"address":     false,
+			"version":     false,
+			"propagation": false,
+		}
+		setFound := func(v string) {
+			require.False(t, found[v], "duplicate env var found")
+			found[v] = true
+		}
+		for _, v := range envVars {
+			switch v {
+			case "GF_VERSION=":
+				setFound("version")
+			case "GF_INSTANCE_OTLP_ADDRESS=127.0.0.1:4317":
+				setFound("address")
+			case "GF_INSTANCE_OTLP_PROPAGATION=":
+				setFound("propagation")
+			}
+		}
+		for k, f := range found {
+			require.Truef(t, f, "%q env var not found", k)
+		}
+	}
+	expNoTracing := func(t *testing.T, envVars []string) {
+		for _, v := range envVars {
+			assert.False(t, strings.HasPrefix(v, "GF_TRACING"), "should not have tracing env var")
+		}
+	}
+
+	for _, tc := range []struct {
+		name string
+		cfg  *config.Cfg
+		exp  func(t *testing.T, envVars []string)
+	}{
+		{
+			name: "disabled",
+			cfg: &config.Cfg{
+				Opentelemetry: config.OpentelemetryCfg{},
+			},
+			exp: expNoTracing,
+		},
+		{
+			name: "otlp no propagation",
+			cfg: &config.Cfg{
+				Opentelemetry: defaultOtelCfg,
+			},
+			exp: expDefaultOtlp,
+		},
+		{
+			name: "otlp propagation",
+			cfg: &config.Cfg{
+				Opentelemetry: config.OpentelemetryCfg{
+					Address:     "127.0.0.1:4317",
+					Propagation: "w3c",
+				},
+			},
+			exp: func(t *testing.T, envVars []string) {
+				assert.Len(t, envVars, 3)
+				assert.Equal(t, "GF_VERSION=", envVars[0])
+				assert.Equal(t, "GF_INSTANCE_OTLP_ADDRESS=127.0.0.1:4317", envVars[1])
+				assert.Equal(t, "GF_INSTANCE_OTLP_PROPAGATION=w3c", envVars[2])
+			},
+		},
+		{
+			name: "disabled on plugin",
+			cfg: &config.Cfg{
+				Opentelemetry: defaultOtelCfg,
+				PluginSettings: setting.PluginSettings{
+					pluginID: map[string]string{"tracing": "false"},
+				},
+			},
+			exp: expNoTracing,
+		},
+		{
+			name: "not disabled on plugin with other plugin settings",
+			cfg: &config.Cfg{
+				Opentelemetry: defaultOtelCfg,
+				PluginSettings: map[string]map[string]string{
+					pluginID: {"some_other_option": "true"},
+				},
+			},
+			exp: expDefaultOtlp,
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			i := &Initializer{
+				cfg: tc.cfg,
+				log: log.NewTestLogger(),
+			}
+			envVars := i.envVars(p)
+			tc.exp(t, envVars)
+		})
+	}
+}
+
 func TestInitializer_getAWSEnvironmentVariables(t *testing.T) {
 
 }

pkg/plugins/manager/manager_integration_test.go

@@ -111,7 +111,8 @@ func TestIntegrationPluginManager(t *testing.T) {
 
 	coreRegistry := coreplugin.ProvideCoreRegistry(am, cw, cm, es, grap, idb, lk, otsdb, pr, tmpo, td, pg, my, ms, graf, phlare, parca)
 
-	pCfg := config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
+	pCfg, err := config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
+	require.NoError(t, err)
 	reg := registry.ProvideService()
 	cdn := pluginscdn.ProvideService(pCfg)