mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
K8s: Add tracing to apiserver startup (#95744)
Co-authored-by: Stephanie Hingtgen <stephanie.hingtgen@grafana.com>
This commit is contained in:
parent
c5178807be
commit
003b2f14db
2
go.mod
2
go.mod
@ -96,6 +96,7 @@ require (
|
||||
// For local development grafana/grafana will always use the local files
|
||||
// Check go.work file for details
|
||||
github.com/grafana/grafana/pkg/promlib v0.0.6 // @grafana/observability-metrics
|
||||
github.com/grafana/grafana/pkg/semconv v0.0.0-20240808213237-f4d2e064f435 // @grafana/grafana-app-platform-squad
|
||||
github.com/grafana/otel-profiling-go v0.5.1 // @grafana/grafana-backend-group
|
||||
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // @grafana/observability-traces-and-profiling
|
||||
github.com/grafana/pyroscope/api v0.3.0 // @grafana/observability-traces-and-profiling
|
||||
@ -479,7 +480,6 @@ require (
|
||||
github.com/dolthub/maphash v0.1.0 // indirect
|
||||
github.com/emirpasic/gods v1.18.1 // indirect
|
||||
github.com/gammazero/deque v0.2.1 // indirect
|
||||
github.com/grafana/grafana/pkg/semconv v0.0.0-20240808213237-f4d2e064f435 // indirect
|
||||
github.com/grafana/sqlds/v4 v4.1.0 // indirect
|
||||
github.com/maypok86/otter v1.2.2 // indirect
|
||||
github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
|
||||
|
35
pkg/modules/tracing/service.go
Normal file
35
pkg/modules/tracing/service.go
Normal file
@ -0,0 +1,35 @@
|
||||
package tracing
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/grafana/dskit/services"
|
||||
"github.com/grafana/grafana/pkg/semconv"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
)
|
||||
|
||||
var _ services.NamedService = &ServiceTracer{}
|
||||
|
||||
// ServiceTracer wraps service.NamedService and adds tracing.
|
||||
// Currently it is limited to the starting -> running state transition.
|
||||
type ServiceTracer struct {
|
||||
services.NamedService
|
||||
tracer trace.Tracer
|
||||
}
|
||||
|
||||
// NewServiceTracer creates a new ServiceTracer.
|
||||
func NewServiceTracer(tracerProvider trace.TracerProvider, service services.NamedService) *ServiceTracer {
|
||||
tracer := tracerProvider.Tracer("pkg/modules/tracing")
|
||||
return &ServiceTracer{NamedService: service, tracer: tracer}
|
||||
}
|
||||
|
||||
func (s *ServiceTracer) StartAsync(ctx context.Context) error {
|
||||
spanCtx, span := s.tracer.Start(ctx, "Service Start", trace.WithAttributes(semconv.GrafanaServiceName(s.ServiceName())))
|
||||
go func() {
|
||||
if err := s.AwaitRunning(spanCtx); err != nil {
|
||||
span.RecordError(err)
|
||||
}
|
||||
span.End()
|
||||
}()
|
||||
return s.NamedService.StartAsync(ctx)
|
||||
}
|
@ -8,17 +8,30 @@ import (
|
||||
|
||||
"github.com/grafana/dskit/services"
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apiserver/pkg/endpoints/responsewriter"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
clientrest "k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
|
||||
|
||||
dataplaneaggregator "github.com/grafana/grafana/pkg/aggregator/apiserver"
|
||||
"github.com/grafana/grafana/pkg/api/routing"
|
||||
"github.com/grafana/grafana/pkg/apimachinery/identity"
|
||||
grafanaresponsewriter "github.com/grafana/grafana/pkg/apiserver/endpoints/responsewriter"
|
||||
"github.com/grafana/grafana/pkg/infra/db"
|
||||
"github.com/grafana/grafana/pkg/infra/kvstore"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/infra/metrics"
|
||||
"github.com/grafana/grafana/pkg/infra/serverlock"
|
||||
"github.com/grafana/grafana/pkg/infra/tracing"
|
||||
"github.com/grafana/grafana/pkg/middleware"
|
||||
"github.com/grafana/grafana/pkg/modules"
|
||||
servicetracing "github.com/grafana/grafana/pkg/modules/tracing"
|
||||
"github.com/grafana/grafana/pkg/plugins"
|
||||
"github.com/grafana/grafana/pkg/registry"
|
||||
"github.com/grafana/grafana/pkg/registry/apis/datasource"
|
||||
@ -36,16 +49,6 @@ import (
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/apistore"
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||
"k8s.io/apiserver/pkg/endpoints/responsewriter"
|
||||
genericapiserver "k8s.io/apiserver/pkg/server"
|
||||
clientrest "k8s.io/client-go/rest"
|
||||
"k8s.io/client-go/tools/clientcmd"
|
||||
aggregatorapiserver "k8s.io/kube-aggregator/pkg/apiserver"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -86,7 +89,7 @@ type RestConfigProvider interface {
|
||||
|
||||
type DirectRestConfigProvider interface {
|
||||
// GetDirectRestConfig returns a k8s client configuration that will use the same
|
||||
// logged logged in user as the current request context. This is useful when
|
||||
// logged in user as the current request context. This is useful when
|
||||
// creating clients that map legacy API handlers to k8s backed services
|
||||
GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Config
|
||||
|
||||
@ -95,15 +98,15 @@ type DirectRestConfigProvider interface {
|
||||
}
|
||||
|
||||
type service struct {
|
||||
*services.BasicService
|
||||
services.NamedService
|
||||
|
||||
options *grafanaapiserveroptions.Options
|
||||
restConfig *clientrest.Config
|
||||
|
||||
cfg *setting.Cfg
|
||||
features featuremgmt.FeatureToggles
|
||||
log log.Logger
|
||||
|
||||
startedCh chan struct{}
|
||||
stopCh chan struct{}
|
||||
stoppedCh chan error
|
||||
|
||||
@ -142,10 +145,10 @@ func ProvideService(
|
||||
unified resource.ResourceClient,
|
||||
) (*service, error) {
|
||||
s := &service{
|
||||
log: log.New(modules.GrafanaAPIServer),
|
||||
cfg: cfg,
|
||||
features: features,
|
||||
rr: rr,
|
||||
startedCh: make(chan struct{}),
|
||||
stopCh: make(chan struct{}),
|
||||
builders: []builder.APIGroupBuilder{},
|
||||
authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService),
|
||||
@ -161,17 +164,23 @@ func ProvideService(
|
||||
unified: unified,
|
||||
}
|
||||
// This will be used when running as a dskit service
|
||||
s.BasicService = services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer)
|
||||
service := services.NewBasicService(s.start, s.running, nil).WithName(modules.GrafanaAPIServer)
|
||||
s.NamedService = servicetracing.NewServiceTracer(tracing.GetTracerProvider(), service)
|
||||
|
||||
// TODO: this is very hacky
|
||||
// We need to register the routes in ProvideService to make sure
|
||||
// the routes are registered before the Grafana HTTP server starts.
|
||||
proxyHandler := func(k8sRoute routing.RouteRegister) {
|
||||
handler := func(c *contextmodel.ReqContext) {
|
||||
<-s.startedCh
|
||||
if err := s.NamedService.AwaitRunning(c.Req.Context()); err != nil {
|
||||
c.Resp.WriteHeader(http.StatusInternalServerError)
|
||||
_, _ = c.Resp.Write([]byte(http.StatusText(http.StatusInternalServerError)))
|
||||
return
|
||||
}
|
||||
|
||||
if s.handler == nil {
|
||||
c.Resp.WriteHeader(404)
|
||||
_, _ = c.Resp.Write([]byte("Not found"))
|
||||
c.Resp.WriteHeader(http.StatusNotFound)
|
||||
_, _ = c.Resp.Write([]byte(http.StatusText(http.StatusNotFound)))
|
||||
return
|
||||
}
|
||||
|
||||
@ -203,7 +212,7 @@ func ProvideService(
|
||||
}
|
||||
|
||||
func (s *service) GetRestConfig(ctx context.Context) *clientrest.Config {
|
||||
if err := s.BasicService.AwaitRunning(ctx); err != nil {
|
||||
if err := s.NamedService.AwaitRunning(ctx); err != nil {
|
||||
return nil
|
||||
}
|
||||
return s.restConfig
|
||||
@ -215,11 +224,11 @@ func (s *service) IsDisabled() bool {
|
||||
|
||||
// Run is an adapter for the BackgroundService interface.
|
||||
func (s *service) Run(ctx context.Context) error {
|
||||
if err := s.BasicService.StartAsync(ctx); err != nil {
|
||||
if err := s.NamedService.StartAsync(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := s.BasicService.AwaitRunning(ctx); err != nil {
|
||||
if err := s.NamedService.AwaitRunning(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.AwaitTerminated(ctx)
|
||||
@ -231,8 +240,6 @@ func (s *service) RegisterAPI(b builder.APIGroupBuilder) {
|
||||
|
||||
// nolint:gocyclo
|
||||
func (s *service) start(ctx context.Context) error {
|
||||
defer close(s.startedCh)
|
||||
|
||||
// Get the list of groups the server will support
|
||||
builders := s.builders
|
||||
|
||||
@ -493,7 +500,9 @@ func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Co
|
||||
return &clientrest.Config{
|
||||
Transport: &roundTripperFunc{
|
||||
fn: func(req *http.Request) (*http.Response, error) {
|
||||
<-s.startedCh
|
||||
if err := s.NamedService.AwaitRunning(req.Context()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ctx := identity.WithRequester(req.Context(), c.SignedInUser)
|
||||
wrapped := grafanaresponsewriter.WrapHandler(s.handler)
|
||||
return wrapped(req.WithContext(ctx))
|
||||
@ -503,7 +512,9 @@ func (s *service) GetDirectRestConfig(c *contextmodel.ReqContext) *clientrest.Co
|
||||
}
|
||||
|
||||
func (s *service) DirectlyServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
<-s.startedCh
|
||||
if err := s.NamedService.AwaitRunning(r.Context()); err != nil {
|
||||
return
|
||||
}
|
||||
s.handler.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user