Add interface Tracer, add Opentelemetry (#41963)

* Add interface Tracer, add Opentelemetry

* Fix lint

* Fix failing tests and return error if config not parsed fo opentelemetry

* Update defaults.ini

Add comment with jaeger url

* go mod tidy

* Remove comments that are not needed

* Move OpentracingSpan to tracing.go

* Add opentelemetry to sample.ini

conf/defaults.ini

@@ -914,6 +914,10 @@ zipkin_propagation = false
 # Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
 disable_shared_zipkin_spans = false
 
+[tracing.opentelemetry.jaeger]
+# jaeger destination (ex http://localhost:14268/api/traces)
+address =
+
 #################################### External Image Storage ##############
 [external_image_storage]
 # Used for uploading images to public servers so they can be included in slack/email messages.

conf/sample.ini

@@ -891,6 +891,10 @@
 # Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
 ;disable_shared_zipkin_spans = false
 
+[tracing.opentelemetry.jaeger]
+# jaeger destination (ex http://localhost:14268/api/traces)
+; address = http://localhost:14268/api/traces
+
 #################################### External image storage ##########################
 [external_image_storage]
 # Used for uploading images to public servers so they can be included in slack/email messages.

go.mod

@@ -101,10 +101,10 @@ require (
 	github.com/yudai/gojsondiff v1.0.0
 	go.opentelemetry.io/collector v0.31.0
 	go.opentelemetry.io/collector/model v0.31.0
-	go.opentelemetry.io/otel v1.0.0
+	go.opentelemetry.io/otel v1.2.0
 	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
 	go.opentelemetry.io/otel/sdk v1.0.0
-	go.opentelemetry.io/otel/trace v1.0.0
+	go.opentelemetry.io/otel/trace v1.2.0
 	golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
 	golang.org/x/exp v0.0.0-20210220032938-85be41e4509f // indirect
 	golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f
@@ -133,7 +133,6 @@ require (
 	cloud.google.com/go v0.94.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
 	github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
@@ -189,7 +188,6 @@ require (
 	github.com/google/flatbuffers v2.0.0+incompatible // indirect
 	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
-	github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect
 	github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
@@ -258,6 +256,11 @@ require (
 	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
 )
 
+require (
+	github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect
+	github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect
+)
+
 // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream
 replace github.com/crewjam/saml => github.com/grafana/saml v0.0.0-20211007135653-aed1b2edd86b
 

go.sum

@@ -2470,8 +2470,9 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.21.0/go.mod h1:
 go.opentelemetry.io/contrib/zpages v0.0.0-20210722161726-7668016acb73/go.mod h1:NAkejuYm41lpyL43Fu1XdnCOYxN5NVV80/MJ03JQ/X8=
 go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0=
 go.opentelemetry.io/otel v1.0.0-RC1/go.mod h1:x9tRa9HK4hSSq7jf2TKbqFbtt58/TGk0f9XiEYISI1I=
-go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI=
 go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg=
+go.opentelemetry.io/otel v1.2.0 h1:YOQDvxO1FayUcT9MIhJhgMyNO1WqoduiyvQHzGN0kUQ=
+go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I=
 go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U=
 go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ=
 go.opentelemetry.io/otel/internal/metric v0.21.0/go.mod h1:iOfAaY2YycsXfYD4kaRSbLx2LKmfpKObWBEv9QK5zFo=
@@ -2481,8 +2482,9 @@ go.opentelemetry.io/otel/sdk v1.0.0-RC1/go.mod h1:kj6yPn7Pgt5ByRuwesbaWcRLA+V7BS
 go.opentelemetry.io/otel/sdk v1.0.0 h1:BNPMYUONPNbLneMttKSjQhOTlFLOD9U22HNG1KrIN2Y=
 go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM=
 go.opentelemetry.io/otel/trace v1.0.0-RC1/go.mod h1:86UHmyHWFEtWjfWPSbu0+d0Pf9Q6e1U+3ViBOc+NXAg=
-go.opentelemetry.io/otel/trace v1.0.0 h1:TSBr8GTEtKevYMG/2d21M989r5WJYVimhTHBKVEZuh4=
 go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs=
+go.opentelemetry.io/otel/trace v1.2.0 h1:Ys3iqbqZhcf28hHzrm5WAquMkDHNZTUkw7KHbuNjej0=
+go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.starlark.net v0.0.0-20200901195727-6e684ef5eeee/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

pkg/api/http_server.go

@@ -110,7 +110,7 @@ type HTTPServer struct {
 	SecretsService            secrets.Service
 	DataSourcesService        *datasources.Service
 	cleanUpService            *cleanup.CleanUpService
-	tracingService            *tracing.TracingService
+	tracingService            tracing.Tracer
 	internalMetricsSvc        *metrics.InternalMetricsService
 	updateChecker             *updatechecker.Service
 	searchUsersService        searchusers.Service
@@ -136,7 +136,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	contextHandler *contexthandler.ContextHandler,
 	schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG,
 	libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service,
-	notificationService *notifications.NotificationService, tracingService *tracing.TracingService,
+	notificationService *notifications.NotificationService, tracingService tracing.Tracer,
 	internalMetricsSvc *metrics.InternalMetricsService, quotaService *quota.QuotaService,
 	socialService social.Service, oauthTokenService oauthtoken.OAuthTokenService,
 	encryptionService encryption.Internal, updateChecker *updatechecker.Service, searchUsersService searchusers.Service,

pkg/infra/tracing/opentelemetry_tracing.go

@@ -0,0 +1,123 @@
+package tracing
+
+import (
+	"context"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/setting"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/sdk/resource"
+	tracesdk "go.opentelemetry.io/otel/sdk/trace"
+	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	trace "go.opentelemetry.io/otel/trace"
+)
+
+type Tracer interface {
+	Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span)
+	Run(context.Context) error
+}
+
+type Span interface {
+	End()
+	SetAttributes(kv ...attribute.KeyValue)
+}
+
+var (
+	GlobalTracer trace.Tracer
+)
+
+type OpentelemetryTracingService struct {
+	enabled bool
+	address string
+	log     log.Logger
+
+	tracerProvider *tracesdk.TracerProvider
+
+	Cfg *setting.Cfg
+}
+
+type OpentelemetrySpan struct {
+	span trace.Span
+}
+
+func (ots *OpentelemetryTracingService) parseSettingsOpentelemetry() error {
+	section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
+	if err != nil {
+		return err
+	}
+
+	ots.address = section.Key("address").MustString("")
+	if ots.address != "" {
+		ots.enabled = true
+	}
+
+	return nil
+}
+
+func (ots *OpentelemetryTracingService) initTracerProvider() (*tracesdk.TracerProvider, error) {
+	// Create the Jaeger exporter
+	exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
+	if err != nil {
+		return nil, err
+	}
+
+	tp := tracesdk.NewTracerProvider(
+		tracesdk.WithBatcher(exp),
+		tracesdk.WithResource(resource.NewWithAttributes(
+			semconv.SchemaURL,
+			semconv.ServiceNameKey.String("grafana"),
+			attribute.String("environment", "production"),
+		)),
+	)
+
+	return tp, nil
+}
+
+func (ots *OpentelemetryTracingService) initOpentelemetryTracer() error {
+	tp, err := ots.initTracerProvider()
+	if err != nil {
+		return err
+	}
+	// Register our TracerProvider as the global so any imported
+	// instrumentation in the future will default to using it
+	// only if tracing is enabled
+	if ots.enabled {
+		otel.SetTracerProvider(tp)
+	}
+
+	ots.tracerProvider = tp
+	GlobalTracer = otel.GetTracerProvider().Tracer("component-main")
+
+	return nil
+}
+
+func (ots *OpentelemetryTracingService) Run(ctx context.Context) error {
+	<-ctx.Done()
+
+	ots.log.Info("Closing tracing")
+	ctxShutdown, cancel := context.WithTimeout(ctx, time.Second*5)
+	defer cancel()
+
+	if err := ots.tracerProvider.Shutdown(ctxShutdown); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (ots *OpentelemetryTracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
+	ctx, span := GlobalTracer.Start(ctx, spanName)
+	oSpan := OpentelemetrySpan{span: span}
+	return ctx, oSpan
+}
+
+func (s OpentelemetrySpan) End() {
+	s.span.End()
+}
+
+func (s OpentelemetrySpan) SetAttributes(kv ...attribute.KeyValue) {
+	s.span.SetAttributes(kv...)
+}

pkg/infra/tracing/tracing.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/setting"
+	"go.opentelemetry.io/otel/attribute"
+	trace "go.opentelemetry.io/otel/trace"
 
 	opentracing "github.com/opentracing/opentracing-go"
 	jaegercfg "github.com/uber/jaeger-client-go/config"
@@ -20,11 +22,12 @@ const (
 	envJaegerAgentPort = "JAEGER_AGENT_PORT"
 )
 
-func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
+func ProvideService(cfg *setting.Cfg) (Tracer, error) {
 	ts := &TracingService{
 		Cfg: cfg,
 		log: log.New("tracing"),
 	}
+
 	if err := ts.parseSettings(); err != nil {
 		return nil, err
 	}
@@ -33,7 +36,16 @@ func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
 		return ts, ts.initGlobalTracer()
 	}
 
-	return ts, nil
+	ots := &OpentelemetryTracingService{
+		Cfg: cfg,
+		log: log.New("tracing"),
+	}
+
+	if err := ots.parseSettingsOpentelemetry(); err != nil {
+		return nil, err
+	}
+
+	return ots, ots.initOpentelemetryTracer()
 }
 
 type TracingService struct {
@@ -51,6 +63,10 @@ type TracingService struct {
 	Cfg *setting.Cfg
 }
 
+type OpentracingSpan struct {
+	span opentracing.Span
+}
+
 func (ts *TracingService) parseSettings() error {
 	var section, err = ts.Cfg.Raw.GetSection("tracing.jaeger")
 	if err != nil {
@@ -150,6 +166,24 @@ func (ts *TracingService) Run(ctx context.Context) error {
 	return nil
 }
 
+func (ts *TracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
+	span, ctx := opentracing.StartSpanFromContext(ctx, spanName)
+	oSpan := OpentracingSpan{
+		span: span,
+	}
+	return ctx, oSpan
+}
+
+func (s OpentracingSpan) End() {
+	s.span.Finish()
+}
+
+func (s OpentracingSpan) SetAttributes(kv ...attribute.KeyValue) {
+	for k, v := range kv {
+		s.span.SetTag(fmt.Sprint(k), v)
+	}
+}
+
 func splitTagSettings(input string) map[string]string {
 	res := map[string]string{}
 

pkg/server/backgroundsvcs/background_services.go

@@ -45,7 +45,7 @@ func ProvideBackgroundServiceRegistry(
 	rendering *rendering.RenderingService, tokenService models.UserTokenBackgroundService,
 	provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, pm *manager.PluginManager,
 	metrics *metrics.InternalMetricsService, usageStats *uss.UsageStats, updateChecker *updatechecker.Service,
-	tracing *tracing.TracingService, remoteCache *remotecache.RemoteCache,
+	tracing tracing.Tracer, remoteCache *remotecache.RemoteCache,
 	// Need to make sure these are initialized, is there a better place to put them?
 	_ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service,
 	_ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service,