K8s: Add tracing to apiserver startup (#95744)

Co-authored-by: Stephanie Hingtgen <stephanie.hingtgen@grafana.com>

go.mod

@@ -96,6 +96,7 @@ require (
 	// For local development grafana/grafana will always use the local files
 	// Check go.work file for details
 	github.com/grafana/grafana/pkg/promlib v0.0.6 // @grafana/observability-metrics
+	github.com/grafana/grafana/pkg/semconv v0.0.0-20240808213237-f4d2e064f435 // @grafana/grafana-app-platform-squad
 	github.com/grafana/otel-profiling-go v0.5.1 // @grafana/grafana-backend-group
 	github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // @grafana/observability-traces-and-profiling
 	github.com/grafana/pyroscope/api v0.3.0 // @grafana/observability-traces-and-profiling
@@ -479,7 +480,6 @@ require (
 	github.com/dolthub/maphash v0.1.0 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/gammazero/deque v0.2.1 // indirect
-	github.com/grafana/grafana/pkg/semconv v0.0.0-20240808213237-f4d2e064f435 // indirect
 	github.com/grafana/sqlds/v4 v4.1.0 // indirect
 	github.com/maypok86/otter v1.2.2 // indirect
 	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect

pkg/modules/tracing/service.go

@@ -0,0 +1,35 @@
+package tracing
+
+import (
+	"context"
+
+	"github.com/grafana/dskit/services"
+	"github.com/grafana/grafana/pkg/semconv"
+	"go.opentelemetry.io/otel/trace"
+)
+
+var _ services.NamedService = &ServiceTracer{}
+
+// ServiceTracer wraps service.NamedService and adds tracing.
+// Currently it is limited to the starting -> running state transition.
+type ServiceTracer struct {
+	services.NamedService
+	tracer trace.Tracer
+}
+
+// NewServiceTracer creates a new ServiceTracer.
+func NewServiceTracer(tracerProvider trace.TracerProvider, service services.NamedService) *ServiceTracer {
+	tracer := tracerProvider.Tracer("pkg/modules/tracing")
+	return &ServiceTracer{NamedService: service, tracer: tracer}
+}
+
+func (s *ServiceTracer) StartAsync(ctx context.Context) error {
+	spanCtx, span := s.tracer.Start(ctx, "Service Start", trace.WithAttributes(semconv.GrafanaServiceName(s.ServiceName())))
+	go func() {
+		if err := s.AwaitRunning(spanCtx); err != nil {
+			span.RecordError(err)
+		}
+		span.End()
+	}()
+	return s.NamedService.StartAsync(ctx)
+}

pkg/services/apiserver/service.go

@@ -8,17 +8,30 @@ import (
 
 	"github.com/grafana/dskit/services"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/prometheus/client_golang/prometheus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apiserver/pkg/endpoints/responsewriter"
+	genericapiserver "k8s.io/apiserver/pkg/server"
+	clientrest "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
+
 	dataplaneaggregator "github.com/grafana/grafana/pkg/aggregator/apiserver"
 	"github.com/grafana/grafana/pkg/api/routing"
 	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	grafanaresponsewriter "github.com/grafana/grafana/pkg/apiserver/endpoints/responsewriter"
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/kvstore"
+	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/infra/serverlock"
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/modules"
+	servicetracing "github.com/grafana/grafana/pkg/modules/tracing"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/registry"
 	"github.com/grafana/grafana/pkg/registry/apis/datasource"
@@ -36,16 +49,6 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/storage/unified/apistore"
 	"github.com/grafana/grafana/pkg/storage/unified/resource"
-	"github.com/prometheus/client_golang/prometheus"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apimachinery/pkg/runtime/serializer"
-	"k8s.io/apiserver/pkg/endpoints/responsewriter"
-	genericapiserver "k8s.io/apiserver/pkg/server"
-	clientrest "k8s.io/client-go/rest"
-	"k8s.io/client-go/tools/clientcmd"
-	aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
 )
 
 var (
@@ -86,7 +89,7 @@ type RestConfigProvider interface {
 
 type DirectRestConfigProvider interface {
 	// GetDirectRestConfig returns a k8s client configuration that will use the same
-	// logged logged in user as the current request context.  This is useful when
+	// logged in user as the current request context.  This is useful when
 	// creating clients that map legacy API handlers to k8s backed services
 	GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Config
 
@@ -95,15 +98,15 @@ type DirectRestConfigProvider interface {
 }
 
 type service struct {
-	*services.BasicService
+	services.NamedService
 
 	options    *grafanaapiserveroptions.Options
 	restConfig *clientrest.Config
 
 	cfg      *setting.Cfg
 	features featuremgmt.FeatureToggles
+	log      log.Logger
 
-	startedCh chan struct{}
 	stopCh    chan struct{}
 	stoppedCh chan error
 
@@ -142,10 +145,10 @@ func ProvideService(
 	unified resource.ResourceClient,
 ) (*service, error) {
 	s := &service{
+		log:               log.New(modules.GrafanaAPIServer),
 		cfg:               cfg,
 		features:          features,
 		rr:                rr,
-		startedCh:         make(chan struct{}),
 		stopCh:            make(chan struct{}),
 		builders:          []builder.APIGroupBuilder{},
 		authorizer:        authorizer.NewGrafanaAuthorizer(cfg, orgService),
@@ -161,17 +164,23 @@ func ProvideService(
 		unified:           unified,
 	}
 	// This will be used when running as a dskit service
-	s.BasicService = services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer)
+	service := services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer)
+	s.NamedService = servicetracing.NewServiceTracer(tracing.GetTracerProvider(), service)
 
 	// TODO: this is very hacky
 	// We need to register the routes in ProvideService to make sure
 	// the routes are registered before the Grafana HTTP server starts.
 	proxyHandler := func(k8sRoute routing.RouteRegister) {
 		handler := func(c *contextmodel.ReqContext) {
-			<-s.startedCh
+			if err := s.NamedService.AwaitRunning(c.Req.Context()); err != nil {
+				c.Resp.WriteHeader(http.StatusInternalServerError)
+				_, _ = c.Resp.Write([]byte(http.StatusText(http.StatusInternalServerError)))
+				return
+			}
+
 			if s.handler == nil {
-				c.Resp.WriteHeader(404)
-				_, _ = c.Resp.Write([]byte("Not found"))
+				c.Resp.WriteHeader(http.StatusNotFound)
+				_, _ = c.Resp.Write([]byte(http.StatusText(http.StatusNotFound)))
 				return
 			}
 
@@ -203,7 +212,7 @@ func ProvideService(
 }
 
 func (s *service) GetRestConfig(ctx context.Context) *clientrest.Config {
-	if err := s.BasicService.AwaitRunning(ctx); err != nil {
+	if err := s.NamedService.AwaitRunning(ctx); err != nil {
 		return nil
 	}
 	return s.restConfig
@@ -215,11 +224,11 @@ func (s *service) IsDisabled() bool {
 
 // Run is an adapter for the BackgroundService interface.
 func (s *service) Run(ctx context.Context) error {
-	if err := s.BasicService.StartAsync(ctx); err != nil {
+	if err := s.NamedService.StartAsync(ctx); err != nil {
 		return err
 	}
 
-	if err := s.BasicService.AwaitRunning(ctx); err != nil {
+	if err := s.NamedService.AwaitRunning(ctx); err != nil {
 		return err
 	}
 	return s.AwaitTerminated(ctx)
@@ -231,8 +240,6 @@ func (s *service) RegisterAPI(b builder.APIGroupBuilder) {
 
 // nolint:gocyclo
 func (s *service) start(ctx context.Context) error {
-	defer close(s.startedCh)
-
 	// Get the list of groups the server will support
 	builders := s.builders
 
@@ -493,7 +500,9 @@ func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Co
 	return &clientrest.Config{
 		Transport: &roundTripperFunc{
 			fn: func(req *http.Request) (*http.Response, error) {
-				<-s.startedCh
+				if err := s.NamedService.AwaitRunning(req.Context()); err != nil {
+					return nil, err
+				}
 				ctx := identity.WithRequester(req.Context(), c.SignedInUser)
 				wrapped := grafanaresponsewriter.WrapHandler(s.handler)
 				return wrapped(req.WithContext(ctx))
@@ -503,7 +512,9 @@ func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Co
 }
 
 func (s *service) DirectlyServeHTTP(w http.ResponseWriter, r *http.Request) {
-	<-s.startedCh
+	if err := s.NamedService.AwaitRunning(r.Context()); err != nil {
+		return
+	}
 	s.handler.ServeHTTP(w, r)
 }