Tracing: Add new [tracing.opentelemetry] custom_attributes config setting (#54110)

* tracing: Add new [tracing.opentelemetry] custom_attributes config setting

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>

* Fix typos in config

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>

* Return error when custom_attributes contains malformed entries

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>

Signed-off-by: Dave Henderson <dave.henderson@grafana.com>

conf/defaults.ini

@@ -1039,6 +1039,11 @@ zipkin_propagation = false
 # Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
 disable_shared_zipkin_spans = false
 
+[tracing.opentelemetry]
+
+# attributes that will always be included in when creating new spans. ex (key1:value1,key2:value2)
+custom_attributes =
+
 [tracing.opentelemetry.jaeger]
 # jaeger destination (ex http://localhost:14268/api/traces)
 address =

conf/sample.ini

@@ -1008,6 +1008,10 @@
 # Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
 ;disable_shared_zipkin_spans = false
 
+[tracing.opentelemetry]
+# attributes that will always be included in when creating new spans. ex (key1:value1,key2:value2)
+;custom_attributes = key1:value1,key2:value2
+
 [tracing.opentelemetry.jaeger]
 # jaeger destination (ex http://localhost:14268/api/traces)
 ; address = http://localhost:14268/api/traces

docs/sources/setup-grafana/configure-grafana/_index.md

@@ -1571,6 +1571,18 @@ Setting this to `true` turns off shared RPC spans. Leaving this available is the
 
 <hr>
 
+## [tracing.opentelemetry]
+
+Configure general parameters shared between OpenTelemetry providers.
+
+### custom_attributes
+
+Comma-separated list of attributes to include in all new spans, such as `key1:value1,key2:value2`.
+
+Can be set with the environment variable `OTEL_RESOURCE_ATTRIBUTES` (use `=` instead of `:` with the environment variable).
+
+<hr>
+
 ## [tracing.opentelemetry.jaeger]
 
 Configure Grafana's Jaeger client for distributed tracing.

pkg/infra/tracing/opentelemetry_tracing.go

@@ -2,7 +2,9 @@ package tracing
 
 import (
 	"context"
+	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/go-kit/log/level"
@@ -48,10 +50,11 @@ type Span interface {
 }
 
 type Opentelemetry struct {
-	enabled     string
-	address     string
-	propagation string
-	log         log.Logger
+	enabled       string
+	address       string
+	propagation   string
+	customAttribs []attribute.KeyValue
+	log           log.Logger
 
 	tracerProvider tracerProvider
 	tracer         trace.Tracer
@@ -89,7 +92,17 @@ func (noopTracerProvider) Shutdown(ctx context.Context) error {
 }
 
 func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
-	section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
+	section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry")
+	if err != nil {
+		return err
+	}
+
+	ots.customAttribs, err = splitCustomAttribs(section.Key("custom_attributes").MustString(""))
+	if err != nil {
+		return err
+	}
+
+	section, err = ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
 	if err != nil {
 		return err
 	}
@@ -115,6 +128,22 @@ func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
 	return nil
 }
 
+func splitCustomAttribs(s string) ([]attribute.KeyValue, error) {
+	res := []attribute.KeyValue{}
+
+	attribs := strings.Split(s, ",")
+	for _, v := range attribs {
+		parts := strings.SplitN(v, ":", 2)
+		if len(parts) > 1 {
+			res = append(res, attribute.String(parts[0], parts[1]))
+		} else if v != "" {
+			return nil, fmt.Errorf("custom attribute malformed - must be in 'key:value' form: %q", v)
+		}
+	}
+
+	return res, nil
+}
+
 func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
 	// Create the Jaeger exporter
 	exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
@@ -122,13 +151,23 @@ func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider,
 		return nil, err
 	}
 
-	tp := tracesdk.NewTracerProvider(
-		tracesdk.WithBatcher(exp),
-		tracesdk.WithResource(resource.NewWithAttributes(
-			semconv.SchemaURL,
+	res, err := resource.New(
+		context.Background(),
+		resource.WithAttributes(
+			// TODO: why are these attributes different from ones added to the
+			// OTLP provider?
 			semconv.ServiceNameKey.String("grafana"),
 			attribute.String("environment", "production"),
-		)),
+		),
+		resource.WithAttributes(ots.customAttribs...),
+	)
+	if err != nil {
+		return nil, err
+	}
+
+	tp := tracesdk.NewTracerProvider(
+		tracesdk.WithBatcher(exp),
+		tracesdk.WithResource(res),
 	)
 
 	return tp, nil
@@ -147,6 +186,7 @@ func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, er
 			semconv.ServiceNameKey.String("grafana"),
 			semconv.ServiceVersionKey.String(version.Version),
 		),
+		resource.WithAttributes(ots.customAttribs...),
 		resource.WithProcessRuntimeDescription(),
 		resource.WithTelemetrySDK(),
 	)

pkg/infra/tracing/optentelemetry_tracing_test.go

@@ -0,0 +1,88 @@
+package tracing
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana/pkg/setting"
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/otel/attribute"
+)
+
+func TestSplitCustomAttribs(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected []attribute.KeyValue
+	}{
+		{
+			input:    "key1:value:1",
+			expected: []attribute.KeyValue{attribute.String("key1", "value:1")},
+		},
+		{
+			input: "key1:value1,key2:value2",
+			expected: []attribute.KeyValue{
+				attribute.String("key1", "value1"),
+				attribute.String("key2", "value2"),
+			},
+		},
+		{
+			input:    "",
+			expected: []attribute.KeyValue{},
+		},
+	}
+
+	for _, test := range tests {
+		attribs, err := splitCustomAttribs(test.input)
+		assert.NoError(t, err)
+		assert.EqualValues(t, test.expected, attribs)
+	}
+}
+
+func TestSplitCustomAttribs_Malformed(t *testing.T) {
+	tests := []struct {
+		input    string
+		expected []attribute.KeyValue
+	}{
+		{input: "key1=value1"},
+		{input: "key1"},
+	}
+
+	for _, test := range tests {
+		_, err := splitCustomAttribs(test.input)
+		assert.Error(t, err)
+	}
+}
+
+func TestOptentelemetry_ParseSettingsOpentelemetry(t *testing.T) {
+	cfg := setting.NewCfg()
+	otel := &Opentelemetry{Cfg: cfg}
+
+	otelsect := cfg.Raw.Section("tracing.opentelemetry")
+	jaegersect := cfg.Raw.Section("tracing.opentelemetry.jaeger")
+	otlpsect := cfg.Raw.Section("tracing.opentelemetry.otlp")
+
+	assert.NoError(t, otel.parseSettingsOpentelemetry())
+	assert.Equal(t, noopExporter, otel.enabled)
+
+	otelsect.Key("custom_attributes")
+	assert.NoError(t, otel.parseSettingsOpentelemetry())
+	assert.Empty(t, otel.customAttribs)
+
+	otelsect.Key("custom_attributes").SetValue("key1:value1,key2:value2")
+	assert.NoError(t, otel.parseSettingsOpentelemetry())
+	expected := []attribute.KeyValue{
+		attribute.String("key1", "value1"),
+		attribute.String("key2", "value2"),
+	}
+	assert.Equal(t, expected, otel.customAttribs)
+
+	jaegersect.Key("address").SetValue("somehost:6831")
+	assert.NoError(t, otel.parseSettingsOpentelemetry())
+	assert.Equal(t, "somehost:6831", otel.address)
+	assert.Equal(t, jaegerExporter, otel.enabled)
+
+	jaegersect.Key("address").SetValue("")
+	otlpsect.Key("address").SetValue("somehost:4317")
+	assert.NoError(t, otel.parseSettingsOpentelemetry())
+	assert.Equal(t, "somehost:4317", otel.address)
+	assert.Equal(t, otlpExporter, otel.enabled)
+}