IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Plugins: Support for distributed tracing in backend plugins SDK (#63714)

Browse Source 
* Tracing: Pass OTLP address and propagation format to plugins

* Fix unit tests

* Fix indentation

* Fix plugin manager integration tests

* Goimports

* Pass plugin version to plugins

* Do not add GF_PLUGIN_VERSION if plugin version is not set, add tests

* Allow disabling plugins distributed tracing on a per-plugin basis

* Moved disabled plugins to tracing.opentelemetry config section

* Pre-allocate DisabledPlugins map to the correct size

* Moved disable tracing setting flags in plugin settings

* Renamed plugin env vars for tracing endpoint and propagation

* Fix plugin initializer tests

* Refactoring: Moved OpentelemetryCfg from pkg/infra to pkg/plugins

* Changed GetSection to Section in parseSettingsOpentelemetry

* Add tests for NewOpentelemetryCfg

* Fix test case names in TestNewOpentelemetryCfg

* OpenTelemetry: Remove redundant error checks
This commit is contained in:
Giuseppe Guerra 2023-03-30 23:31:14 +02:00 committed by GitHub
parent 55947cee8f
commit 09078b14e1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 327 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/api/plugin_resource_test.go
View File

@ -50,6 +50,7 @@ func TestCallResource(t *testing.T) {
require.NoError(t, err)
cfg := setting.NewCfg()
cfg.StaticRootPath = staticRootPath
cfg.IsFeatureToggleEnabled = func(_ string) bool {
return false
@ -58,7 +59,9 @@ func TestCallResource(t *testing.T) {
coreRegistry := coreplugin.ProvideCoreRegistry(nil, &cloudwatch.CloudWatchService{}, nil, nil, nil, nil,
nil, nil, nil, nil, testdatasource.ProvideService(cfg, featuremgmt.WithFeatures()), nil, nil, nil, nil, nil, nil)
pCfg := config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
var pCfg *config.Cfg
pCfg, err = config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
require.NoError(t, err)
reg := registry.ProvideService()
cdn := pluginscdn.ProvideService(pCfg)
l := loader.ProvideService(pCfg, fakes.NewFakeLicensingService(), signature.NewUnsignedAuthorizer(pCfg),

55
pkg/infra/tracing/opentelemetry_tracing.go
View File

@ -36,11 +36,12 @@ const (
)
type Opentelemetry struct {
enabled string
address string
propagation string
Enabled string
Address string
Propagation string
customAttribs []attribute.KeyValue
log log.Logger
log log.Logger
tracerProvider tracerProvider
tracer trace.Tracer
@ -78,39 +79,29 @@ func (noopTracerProvider) Shutdown(ctx context.Context) error {
}
func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry")
if err != nil {
return err
}
section := ots.Cfg.Raw.Section("tracing.opentelemetry")
var err error
ots.customAttribs, err = splitCustomAttribs(section.Key("custom_attributes").MustString(""))
if err != nil {
return err
}
section, err = ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
if err != nil {
return err
}
ots.enabled = noopExporter
section = ots.Cfg.Raw.Section("tracing.opentelemetry.jaeger")
ots.Enabled = noopExporter
ots.address = section.Key("address").MustString("")
ots.propagation = section.Key("propagation").MustString("")
if ots.address != "" {
ots.enabled = jaegerExporter
ots.Address = section.Key("address").MustString("")
ots.Propagation = section.Key("propagation").MustString("")
if ots.Address != "" {
ots.Enabled = jaegerExporter
return nil
}
section, err = ots.Cfg.Raw.GetSection("tracing.opentelemetry.otlp")
if err != nil {
return err
section = ots.Cfg.Raw.Section("tracing.opentelemetry.otlp")
ots.Address = section.Key("address").MustString("")
if ots.Address != "" {
ots.Enabled = otlpExporter
}
ots.address = section.Key("address").MustString("")
if ots.address != "" {
ots.enabled = otlpExporter
}
ots.propagation = section.Key("propagation").MustString("")
ots.Propagation = section.Key("propagation").MustString("")
return nil
}
@ -132,7 +123,7 @@ func splitCustomAttribs(s string) ([]attribute.KeyValue, error) {
func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
// Create the Jaeger exporter
exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.Address)))
if err != nil {
return nil, err
}
@ -160,7 +151,7 @@ func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider,
}
func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.address), otlptracegrpc.WithInsecure())
client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.Address), otlptracegrpc.WithInsecure())
exp, err := otlptrace.New(context.Background(), client)
if err != nil {
return nil, err
@ -201,7 +192,7 @@ func (ots *Opentelemetry) initNoopTracerProvider() (tracerProvider, error) {
func (ots *Opentelemetry) initOpentelemetryTracer() error {
var tp tracerProvider
var err error
switch ots.enabled {
switch ots.Enabled {
case jaegerExporter:
tp, err = ots.initJaegerTracerProvider()
if err != nil {
@ -222,12 +213,12 @@ func (ots *Opentelemetry) initOpentelemetryTracer() error {
// Register our TracerProvider as the global so any imported
// instrumentation in the future will default to using it
// only if tracing is enabled
if ots.enabled != "" {
if ots.Enabled != "" {
otel.SetTracerProvider(tp)
}
propagators := []propagation.TextMapPropagator{}
for _, p := range strings.Split(ots.propagation, ",") {
for _, p := range strings.Split(ots.Propagation, ",") {
switch p {
case w3cPropagator:
propagators = append(propagators, propagation.TraceContext{}, propagation.Baggage{})

10
pkg/infra/tracing/optentelemetry_tracing_test.go
View File

@ -62,7 +62,7 @@ func TestOptentelemetry_ParseSettingsOpentelemetry(t *testing.T) {
otlpsect := cfg.Raw.Section("tracing.opentelemetry.otlp")
assert.NoError(t, otel.parseSettingsOpentelemetry())
assert.Equal(t, noopExporter, otel.enabled)
assert.Equal(t, noopExporter, otel.Enabled)
otelsect.Key("custom_attributes")
assert.NoError(t, otel.parseSettingsOpentelemetry())
@ -78,12 +78,12 @@ func TestOptentelemetry_ParseSettingsOpentelemetry(t *testing.T) {
jaegersect.Key("address").SetValue("somehost:6831")
assert.NoError(t, otel.parseSettingsOpentelemetry())
assert.Equal(t, "somehost:6831", otel.address)
assert.Equal(t, jaegerExporter, otel.enabled)
assert.Equal(t, "somehost:6831", otel.Address)
assert.Equal(t, jaegerExporter, otel.Enabled)
jaegersect.Key("address").SetValue("")
otlpsect.Key("address").SetValue("somehost:4317")
assert.NoError(t, otel.parseSettingsOpentelemetry())
assert.Equal(t, "somehost:4317", otel.address)
assert.Equal(t, otlpExporter, otel.enabled)
assert.Equal(t, "somehost:4317", otel.Address)
assert.Equal(t, otlpExporter, otel.Enabled)
}

2
pkg/infra/tracing/test_helper.go
View File

@ -16,7 +16,7 @@ func InitializeTracerForTest() Tracer {
tp, _ := initTracerProvider(exp)
otel.SetTracerProvider(tp)
ots := &Opentelemetry{propagation: "jaeger,w3c", tracerProvider: tp}
ots := &Opentelemetry{Propagation: "jaeger,w3c", tracerProvider: tp}
_ = ots.initOpentelemetryTracer()
return ots
}

22
pkg/infra/tracing/tracing.go
View File

@ -100,25 +100,35 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) {
}
func parseSettings(cfg *setting.Cfg) (*Opentracing, *Opentelemetry, error) {
ts, err := parseSettingsOpentracing(cfg)
if err != nil {
return ts, nil, err
}
ots, err := ParseSettingsOpentelemetry(cfg)
return ts, ots, err
}
func parseSettingsOpentracing(cfg *setting.Cfg) (*Opentracing, error) {
ts := &Opentracing{
Cfg: cfg,
log: log.New("tracing"),
}
err := ts.parseSettings()
if err != nil {
return ts, nil, err
if err := ts.parseSettings(); err != nil {
return ts, err
}
if ts.enabled {
cfg.Logger.Warn("[Deprecated] the configuration setting 'tracing.jaeger' is deprecated, please use 'tracing.opentelemetry.jaeger' instead")
return ts, nil, nil
}
return ts, nil
}
func ParseSettingsOpentelemetry(cfg *setting.Cfg) (*Opentelemetry, error) {
ots := &Opentelemetry{
Cfg: cfg,
log: log.New("tracing"),
}
err = ots.parseSettingsOpentelemetry()
return ts, ots, err
err := ots.parseSettingsOpentelemetry()
return ots, err
}
type traceKey struct{}

14
pkg/plugins/config/config.go
View File

@ -1,6 +1,7 @@
package config
import (
"fmt"
"strings"
"github.com/grafana/grafana-azure-sdk-go/azsettings"
@ -31,13 +32,15 @@ type Cfg struct {
LogDatasourceRequests bool
PluginsCDNURLTemplate string
Opentelemetry OpentelemetryCfg
}
func ProvideConfig(settingProvider setting.Provider, grafanaCfg *setting.Cfg) *Cfg {
func ProvideConfig(settingProvider setting.Provider, grafanaCfg *setting.Cfg) (*Cfg, error) {
return NewCfg(settingProvider, grafanaCfg)
}
func NewCfg(settingProvider setting.Provider, grafanaCfg *setting.Cfg) *Cfg {
func NewCfg(settingProvider setting.Provider, grafanaCfg *setting.Cfg) (*Cfg, error) {
logger := log.New("plugin.cfg")
aws := settingProvider.Section("aws")
@ -53,6 +56,10 @@ func NewCfg(settingProvider setting.Provider, grafanaCfg *setting.Cfg) *Cfg {
allowedUnsigned = strings.Split(settingProvider.KeyValue("plugins", "allow_loading_unsigned_plugins").Value(), ",")
}
otelCfg, err := NewOpentelemetryCfg(grafanaCfg)
if err != nil {
return nil, fmt.Errorf("new opentelemetry cfg: %w", err)
}
return &Cfg{
log: logger,
PluginsPath: grafanaCfg.PluginsPath,
@ -65,7 +72,8 @@ func NewCfg(settingProvider setting.Provider, grafanaCfg *setting.Cfg) *Cfg {
Azure: grafanaCfg.Azure,
LogDatasourceRequests: grafanaCfg.PluginLogBackendRequests,
PluginsCDNURLTemplate: grafanaCfg.PluginsCDNURLTemplate,
}
Opentelemetry: otelCfg,
}, nil
}
func extractPluginSettings(settingProvider setting.Provider) setting.PluginSettings {

38
pkg/plugins/config/tracing.go Normal file
View File

@ -0,0 +1,38 @@
package config
import (
"fmt"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/setting"
)
const otlpExporter string = "otlp"
// OpentelemetryCfg contains the Opentelemetry address and propagation config values.
// This is used to export the Opentelemetry (OTLP) config without exposing the whole *setting.Cfg.
type OpentelemetryCfg struct {
Address string
Propagation string
}
// IsEnabled returns true if OTLP tracing is enabled (address set)
func (c OpentelemetryCfg) IsEnabled() bool {
return c.Address != ""
}
// NewOpentelemetryCfg creates a new OpentelemetryCfg based on the provided Grafana config.
// If Opentelemetry (OTLP) is disabled, a zero-value OpentelemetryCfg is returned.
func NewOpentelemetryCfg(grafanaCfg *setting.Cfg) (OpentelemetryCfg, error) {
ots, err := tracing.ParseSettingsOpentelemetry(grafanaCfg)
if err != nil {
return OpentelemetryCfg{}, fmt.Errorf("parse settings: %w", err)
}
if ots.Enabled != otlpExporter {
return OpentelemetryCfg{}, nil
}
return OpentelemetryCfg{
Address: ots.Address,
Propagation: ots.Propagation,
}, nil
}

50
pkg/plugins/config/tracing_test.go Normal file
View File

@ -0,0 +1,50 @@
package config
import (
"testing"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestNewOpentelemetryCfg(t *testing.T) {
t.Run("empty", func(t *testing.T) {
cfg := setting.NewCfg()
otelCfg, err := NewOpentelemetryCfg(cfg)
require.NoError(t, err)
assert.False(t, otelCfg.IsEnabled(), "otel should be disabled")
assert.Empty(t, otelCfg.Address)
assert.Empty(t, otelCfg.Propagation)
})
t.Run("enabled", func(t *testing.T) {
for _, tc := range []struct {
name string
propagation string
}{
{"empty", ""},
{"jaeger", "jaeger"},
{"w3c", "w3c"},
{"multiple", "jaeger,w3c"},
} {
t.Run(tc.name, func(t *testing.T) {
const address = "127.0.0.1:4317"
cfg := setting.NewCfg()
otlpSect := cfg.Raw.Section("tracing.opentelemetry.otlp")
otlpSect.Key("address").SetValue(address)
if tc.propagation != "" {
otlpSect.Key("propagation").SetValue(tc.propagation)
}
otelCfg, err := NewOpentelemetryCfg(cfg)
require.NoError(t, err)
assert.True(t, otelCfg.IsEnabled(), "otel should be enabled")
assert.Equal(t, address, otelCfg.Address)
assert.Equal(t, tc.propagation, otelCfg.Propagation)
})
}
})
}

20
pkg/plugins/manager/loader/initializer/initializer.go
View File

@ -51,6 +51,9 @@ func (i *Initializer) envVars(plugin *plugins.Plugin) []string {
hostEnv := []string{
fmt.Sprintf("GF_VERSION=%s", i.cfg.BuildVersion),
}
if plugin.Info.Version != "" {
hostEnv = append(hostEnv, fmt.Sprintf("GF_PLUGIN_VERSION=%s", plugin.Info.Version))
}
if i.license != nil {
hostEnv = append(
@ -64,7 +67,22 @@ func (i *Initializer) envVars(plugin *plugins.Plugin) []string {
hostEnv = append(hostEnv, i.awsEnvVars()...)
hostEnv = append(hostEnv, azsettings.WriteToEnvStr(i.cfg.Azure)...)
return getPluginSettings(plugin.ID, i.cfg).asEnvVar("GF_PLUGIN", hostEnv)
// Tracing
pluginTracingEnabled := true
if v, exists := i.cfg.PluginSettings[plugin.ID]["tracing"]; exists {
pluginTracingEnabled = v != "false"
}
if i.cfg.Opentelemetry.IsEnabled() && pluginTracingEnabled {
hostEnv = append(
hostEnv,
fmt.Sprintf("GF_INSTANCE_OTLP_ADDRESS=%s", i.cfg.Opentelemetry.Address),
fmt.Sprintf("GF_INSTANCE_OTLP_PROPAGATION=%s", i.cfg.Opentelemetry.Propagation),
)
}
ev := getPluginSettings(plugin.ID, i.cfg).asEnvVar("GF_PLUGIN", hostEnv)
return ev
}
func (i *Initializer) awsEnvVars() []string {

158
pkg/plugins/manager/loader/initializer/initializer_test.go
View File

@ -2,9 +2,12 @@ package initializer
import (
"context"
"strings"
"testing"
"github.com/grafana/grafana/pkg/setting"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/backendplugin"
@ -129,6 +132,53 @@ func TestInitializer_Initialize(t *testing.T) {
}
func TestInitializer_envVars(t *testing.T) {
t.Run("version", func(t *testing.T) {
for _, tc := range []struct {
name string
setup func(p *plugins.Plugin)
exp func(t *testing.T, i *Initializer, p *plugins.Plugin)
}{
{
name: "not provided",
setup: func(p *plugins.Plugin) {
p.Info = plugins.Info{}
},
exp: func(t *testing.T, i *Initializer, p *plugins.Plugin) {
for _, k := range i.envVars(p) {
if strings.HasPrefix("GF_PLUGIN_VERSION=", k) {
require.Fail(t, "found unexpected env var GF_PLUGIN_VERSION")
}
}
},
},
{
name: "provided",
setup: func(p *plugins.Plugin) {
p.Info = plugins.Info{Version: "0.1"}
},
exp: func(t *testing.T, i *Initializer, p *plugins.Plugin) {
require.Contains(t, i.envVars(p), "GF_PLUGIN_VERSION=0.1")
},
},
} {
t.Run(tc.name, func(t *testing.T) {
p := &plugins.Plugin{
JSONData: plugins.JSONData{
ID: "test",
Info: plugins.Info{Version: "0.1"},
},
}
tc.setup(p)
i := &Initializer{
cfg: &config.Cfg{},
log: log.NewTestLogger(),
backendProvider: &fakeBackendProvider{plugin: p},
}
tc.exp(t, i, p)
})
}
})
t.Run("backend datasource with license", func(t *testing.T) {
p := &plugins.Plugin{
JSONData: plugins.JSONData{
@ -169,6 +219,114 @@ func TestInitializer_envVars(t *testing.T) {
})
}
func TestInitializer_tracingEnvironmentVariables(t *testing.T) {
const pluginID = "plugin_id"
p := &plugins.Plugin{
JSONData: plugins.JSONData{ID: pluginID},
}
defaultOtelCfg := config.OpentelemetryCfg{
Address: "127.0.0.1:4317",
Propagation: "",
}
expDefaultOtlp := func(t *testing.T, envVars []string) {
found := map[string]bool{
"address": false,
"version": false,
"propagation": false,
}
setFound := func(v string) {
require.False(t, found[v], "duplicate env var found")
found[v] = true
}
for _, v := range envVars {
switch v {
case "GF_VERSION=":
setFound("version")
case "GF_INSTANCE_OTLP_ADDRESS=127.0.0.1:4317":
setFound("address")
case "GF_INSTANCE_OTLP_PROPAGATION=":
setFound("propagation")
}
}
for k, f := range found {
require.Truef(t, f, "%q env var not found", k)
}
}
expNoTracing := func(t *testing.T, envVars []string) {
for _, v := range envVars {
assert.False(t, strings.HasPrefix(v, "GF_TRACING"), "should not have tracing env var")
}
}
for _, tc := range []struct {
name string
cfg *config.Cfg
exp func(t *testing.T, envVars []string)
}{
{
name: "disabled",
cfg: &config.Cfg{
Opentelemetry: config.OpentelemetryCfg{},
},
exp: expNoTracing,
},
{
name: "otlp no propagation",
cfg: &config.Cfg{
Opentelemetry: defaultOtelCfg,
},
exp: expDefaultOtlp,
},
{
name: "otlp propagation",
cfg: &config.Cfg{
Opentelemetry: config.OpentelemetryCfg{
Address: "127.0.0.1:4317",
Propagation: "w3c",
},
},
exp: func(t *testing.T, envVars []string) {
assert.Len(t, envVars, 3)
assert.Equal(t, "GF_VERSION=", envVars[0])
assert.Equal(t, "GF_INSTANCE_OTLP_ADDRESS=127.0.0.1:4317", envVars[1])
assert.Equal(t, "GF_INSTANCE_OTLP_PROPAGATION=w3c", envVars[2])
},
},
{
name: "disabled on plugin",
cfg: &config.Cfg{
Opentelemetry: defaultOtelCfg,
PluginSettings: setting.PluginSettings{
pluginID: map[string]string{"tracing": "false"},
},
},
exp: expNoTracing,
},
{
name: "not disabled on plugin with other plugin settings",
cfg: &config.Cfg{
Opentelemetry: defaultOtelCfg,
PluginSettings: map[string]map[string]string{
pluginID: {"some_other_option": "true"},
},
},
exp: expDefaultOtlp,
},
} {
t.Run(tc.name, func(t *testing.T) {
i := &Initializer{
cfg: tc.cfg,
log: log.NewTestLogger(),
}
envVars := i.envVars(p)
tc.exp(t, envVars)
})
}
}
func TestInitializer_getAWSEnvironmentVariables(t *testing.T) {
}

3
pkg/plugins/manager/manager_integration_test.go
View File

@ -111,7 +111,8 @@ func TestIntegrationPluginManager(t *testing.T) {
coreRegistry := coreplugin.ProvideCoreRegistry(am, cw, cm, es, grap, idb, lk, otsdb, pr, tmpo, td, pg, my, ms, graf, phlare, parca)
pCfg := config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
pCfg, err := config.ProvideConfig(setting.ProvideProvider(cfg), cfg)
require.NoError(t, err)
reg := registry.ProvideService()
cdn := pluginscdn.ProvideService(pCfg)