diff --git a/go.mod b/go.mod index ba6f15ffddb..0a1aab0e588 100644 --- a/go.mod +++ b/go.mod @@ -96,6 +96,7 @@ require ( // For local development grafana/grafana will always use the local files // Check go.work file for details github.com/grafana/grafana/pkg/promlib v0.0.6 // @grafana/observability-metrics + github.com/grafana/grafana/pkg/semconv v0.0.0-20240808213237-f4d2e064f435 // @grafana/grafana-app-platform-squad github.com/grafana/otel-profiling-go v0.5.1 // @grafana/grafana-backend-group github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // @grafana/observability-traces-and-profiling github.com/grafana/pyroscope/api v0.3.0 // @grafana/observability-traces-and-profiling @@ -479,7 +480,6 @@ require ( github.com/dolthub/maphash v0.1.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/gammazero/deque v0.2.1 // indirect - github.com/grafana/grafana/pkg/semconv v0.0.0-20240808213237-f4d2e064f435 // indirect github.com/grafana/sqlds/v4 v4.1.0 // indirect github.com/maypok86/otter v1.2.2 // indirect github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect diff --git a/pkg/modules/tracing/service.go b/pkg/modules/tracing/service.go new file mode 100644 index 00000000000..38fe155edc1 --- /dev/null +++ b/pkg/modules/tracing/service.go @@ -0,0 +1,35 @@ +package tracing + +import ( + "context" + + "github.com/grafana/dskit/services" + "github.com/grafana/grafana/pkg/semconv" + "go.opentelemetry.io/otel/trace" +) + +var _ services.NamedService = &ServiceTracer{} + +// ServiceTracer wraps service.NamedService and adds tracing. +// Currently it is limited to the starting -> running state transition. +type ServiceTracer struct { + services.NamedService + tracer trace.Tracer +} + +// NewServiceTracer creates a new ServiceTracer. +func NewServiceTracer(tracerProvider trace.TracerProvider, service services.NamedService) *ServiceTracer { + tracer := tracerProvider.Tracer("pkg/modules/tracing") + return &ServiceTracer{NamedService: service, tracer: tracer} +} + +func (s *ServiceTracer) StartAsync(ctx context.Context) error { + spanCtx, span := s.tracer.Start(ctx, "Service Start", trace.WithAttributes(semconv.GrafanaServiceName(s.ServiceName()))) + go func() { + if err := s.AwaitRunning(spanCtx); err != nil { + span.RecordError(err) + } + span.End() + }() + return s.NamedService.StartAsync(ctx) +} diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index f11c32b4e93..be56eb1e435 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -8,17 +8,30 @@ import ( "github.com/grafana/dskit/services" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/prometheus/client_golang/prometheus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apiserver/pkg/endpoints/responsewriter" + genericapiserver "k8s.io/apiserver/pkg/server" + clientrest "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" + aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" + dataplaneaggregator "github.com/grafana/grafana/pkg/aggregator/apiserver" "github.com/grafana/grafana/pkg/api/routing" "github.com/grafana/grafana/pkg/apimachinery/identity" grafanaresponsewriter "github.com/grafana/grafana/pkg/apiserver/endpoints/responsewriter" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/kvstore" + "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/infra/serverlock" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/modules" + servicetracing "github.com/grafana/grafana/pkg/modules/tracing" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/registry/apis/datasource" @@ -36,16 +49,6 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/storage/unified/apistore" "github.com/grafana/grafana/pkg/storage/unified/resource" - "github.com/prometheus/client_golang/prometheus" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/runtime/serializer" - "k8s.io/apiserver/pkg/endpoints/responsewriter" - genericapiserver "k8s.io/apiserver/pkg/server" - clientrest "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" - aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver" ) var ( @@ -86,7 +89,7 @@ type RestConfigProvider interface { type DirectRestConfigProvider interface { // GetDirectRestConfig returns a k8s client configuration that will use the same - // logged logged in user as the current request context. This is useful when + // logged in user as the current request context. This is useful when // creating clients that map legacy API handlers to k8s backed services GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Config @@ -95,15 +98,15 @@ type DirectRestConfigProvider interface { } type service struct { - *services.BasicService + services.NamedService options *grafanaapiserveroptions.Options restConfig *clientrest.Config cfg *setting.Cfg features featuremgmt.FeatureToggles + log log.Logger - startedCh chan struct{} stopCh chan struct{} stoppedCh chan error @@ -142,10 +145,10 @@ func ProvideService( unified resource.ResourceClient, ) (*service, error) { s := &service{ + log: log.New(modules.GrafanaAPIServer), cfg: cfg, features: features, rr: rr, - startedCh: make(chan struct{}), stopCh: make(chan struct{}), builders: []builder.APIGroupBuilder{}, authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService), @@ -161,17 +164,23 @@ func ProvideService( unified: unified, } // This will be used when running as a dskit service - s.BasicService = services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer) + service := services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer) + s.NamedService = servicetracing.NewServiceTracer(tracing.GetTracerProvider(), service) // TODO: this is very hacky // We need to register the routes in ProvideService to make sure // the routes are registered before the Grafana HTTP server starts. proxyHandler := func(k8sRoute routing.RouteRegister) { handler := func(c *contextmodel.ReqContext) { - <-s.startedCh + if err := s.NamedService.AwaitRunning(c.Req.Context()); err != nil { + c.Resp.WriteHeader(http.StatusInternalServerError) + _, _ = c.Resp.Write([]byte(http.StatusText(http.StatusInternalServerError))) + return + } + if s.handler == nil { - c.Resp.WriteHeader(404) - _, _ = c.Resp.Write([]byte("Not found")) + c.Resp.WriteHeader(http.StatusNotFound) + _, _ = c.Resp.Write([]byte(http.StatusText(http.StatusNotFound))) return } @@ -203,7 +212,7 @@ func ProvideService( } func (s *service) GetRestConfig(ctx context.Context) *clientrest.Config { - if err := s.BasicService.AwaitRunning(ctx); err != nil { + if err := s.NamedService.AwaitRunning(ctx); err != nil { return nil } return s.restConfig @@ -215,11 +224,11 @@ func (s *service) IsDisabled() bool { // Run is an adapter for the BackgroundService interface. func (s *service) Run(ctx context.Context) error { - if err := s.BasicService.StartAsync(ctx); err != nil { + if err := s.NamedService.StartAsync(ctx); err != nil { return err } - if err := s.BasicService.AwaitRunning(ctx); err != nil { + if err := s.NamedService.AwaitRunning(ctx); err != nil { return err } return s.AwaitTerminated(ctx) @@ -231,8 +240,6 @@ func (s *service) RegisterAPI(b builder.APIGroupBuilder) { // nolint:gocyclo func (s *service) start(ctx context.Context) error { - defer close(s.startedCh) - // Get the list of groups the server will support builders := s.builders @@ -493,7 +500,9 @@ func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Co return &clientrest.Config{ Transport: &roundTripperFunc{ fn: func(req *http.Request) (*http.Response, error) { - <-s.startedCh + if err := s.NamedService.AwaitRunning(req.Context()); err != nil { + return nil, err + } ctx := identity.WithRequester(req.Context(), c.SignedInUser) wrapped := grafanaresponsewriter.WrapHandler(s.handler) return wrapped(req.WithContext(ctx)) @@ -503,7 +512,9 @@ func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Co } func (s *service) DirectlyServeHTTP(w http.ResponseWriter, r *http.Request) { - <-s.startedCh + if err := s.NamedService.AwaitRunning(r.Context()); err != nil { + return + } s.handler.ServeHTTP(w, r) }