diff --git a/pkg/cmd/grafana-cli/commands/conflict_user_command.go b/pkg/cmd/grafana-cli/commands/conflict_user_command.go index 3e607cecca8..b947f88f24b 100644 --- a/pkg/cmd/grafana-cli/commands/conflict_user_command.go +++ b/pkg/cmd/grafana-cli/commands/conflict_user_command.go @@ -79,7 +79,12 @@ func initializeConflictResolver(cmd *utils.ContextCommandLine, f Formatter, ctx } func getSqlStore(cfg *setting.Cfg, features featuremgmt.FeatureToggles) (*sqlstore.SQLStore, error) { - tracer, err := tracing.ProvideService(cfg) + tracingCfg, err := tracing.ProvideTracingConfig(cfg) + if err != nil { + return nil, fmt.Errorf("%v: %w", "failed to initialize tracer config", err) + } + + tracer, err := tracing.ProvideService(tracingCfg) if err != nil { return nil, fmt.Errorf("%v: %w", "failed to initialize tracer service", err) } diff --git a/pkg/cmd/grafana-server/commands/buildinfo.go b/pkg/cmd/grafana-server/commands/buildinfo.go index d7097915f4e..1051ca520bc 100644 --- a/pkg/cmd/grafana-server/commands/buildinfo.go +++ b/pkg/cmd/grafana-server/commands/buildinfo.go @@ -16,7 +16,7 @@ func getBuildstamp(opts ServerOptions) int64 { return buildstampInt64 } -func setBuildInfo(opts ServerOptions) { +func SetBuildInfo(opts ServerOptions) { setting.BuildVersion = opts.Version setting.BuildCommit = opts.Commit setting.EnterpriseBuildCommit = opts.EnterpriseCommit diff --git a/pkg/cmd/grafana-server/commands/cli.go b/pkg/cmd/grafana-server/commands/cli.go index f2d434809d2..313991db613 100644 --- a/pkg/cmd/grafana-server/commands/cli.go +++ b/pkg/cmd/grafana-server/commands/cli.go @@ -98,7 +98,7 @@ func RunServer(opts ServerOptions) error { } }() - setBuildInfo(opts) + SetBuildInfo(opts) checkPrivileges() configOptions := strings.Split(ConfigOverrides, " ") @@ -112,7 +112,7 @@ func RunServer(opts ServerOptions) error { return err } - metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts)) + metrics.SetBuildInformation(metrics.ProvideRegisterer(), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts)) s, err := server.Initialize( cfg, diff --git a/pkg/cmd/grafana-server/commands/target.go b/pkg/cmd/grafana-server/commands/target.go index ae908ac9c18..6293eb78bcd 100644 --- a/pkg/cmd/grafana-server/commands/target.go +++ b/pkg/cmd/grafana-server/commands/target.go @@ -75,7 +75,7 @@ func RunTargetServer(opts ServerOptions) error { } }() - setBuildInfo(opts) + SetBuildInfo(opts) checkPrivileges() configOptions := strings.Split(ConfigOverrides, " ") @@ -89,7 +89,7 @@ func RunTargetServer(opts ServerOptions) error { return err } - metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts)) + metrics.SetBuildInformation(metrics.ProvideRegisterer(), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts)) s, err := server.InitializeModuleServer( cfg, diff --git a/pkg/cmd/grafana/apiserver/apiserver.md b/pkg/cmd/grafana/apiserver/apiserver.md index 27dce737a7d..b84fa70dfe2 100644 --- a/pkg/cmd/grafana/apiserver/apiserver.md +++ b/pkg/cmd/grafana/apiserver/apiserver.md @@ -1,6 +1,6 @@ # grafana apiserver (standalone) -The example-apiserver closely resembles the +The example-apiserver closely resembles the [sample-apiserver](https://github.com/kubernetes/sample-apiserver/tree/master) project in code and thus allows the same [CLI flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) as kube-apiserver. @@ -32,3 +32,25 @@ dummy example.grafana.app/v0alpha1 true DummyResource runtime example.grafana.app/v0alpha1 false RuntimeInfo ``` +### Observability + +Logs, metrics and traces are supported. See `--grafana.log.*`, `--grafana.metrics.*` and `--grafana.tracing.*` flags for details. + +```shell +go run ./pkg/cmd/grafana apiserver \ + --runtime-config=example.grafana.app/v0alpha1=true \ + --help +``` + +For example, to enable debug logs, metrics and traces (using [self-instrumentation](../../../../devenv/docker/blocks/self-instrumentation/readme.md)) use the following: + +```shell +go run ./pkg/cmd/grafana apiserver \ + --runtime-config=example.grafana.app/v0alpha1=true \ + --secure-port=7443 \ + --grafana.log.level=debug \ + --verbosity=10 \ + --grafana.metrics.enable \ + --grafana.tracing.jaeger.address=http://localhost:14268/api/traces \ + --grafana.tracing.sampler-param=1 +``` diff --git a/pkg/cmd/grafana/apiserver/cmd.go b/pkg/cmd/grafana/apiserver/cmd.go index dd63ab8a3d4..d029f65671a 100644 --- a/pkg/cmd/grafana/apiserver/cmd.go +++ b/pkg/cmd/grafana/apiserver/cmd.go @@ -7,6 +7,7 @@ import ( genericapiserver "k8s.io/apiserver/pkg/server" "k8s.io/component-base/cli" + "github.com/grafana/grafana/pkg/cmd/grafana-server/commands" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/server" "github.com/grafana/grafana/pkg/services/apiserver/standalone" @@ -64,27 +65,23 @@ func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{} if err := o.RunAPIServer(config, stopCh); err != nil { return err } + return nil }, } cmd.Flags().StringVar(&runtimeConfig, "runtime-config", "", "A set of key=value pairs that enable or disable built-in APIs.") - if factoryOptions := o.factory.GetOptions(); factoryOptions != nil { - factoryOptions.AddFlags(cmd.Flags()) - } - - o.ExtraOptions.AddFlags(cmd.Flags()) - - // Register standard k8s flags with the command line - o.RecommendedOptions.AddFlags(cmd.Flags()) + o.AddFlags(cmd.Flags()) return cmd } -func RunCLI() int { +func RunCLI(opts commands.ServerOptions) int { stopCh := genericapiserver.SetupSignalHandler() + commands.SetBuildInfo(opts) + options := newAPIServerOptions(os.Stdout, os.Stderr) cmd := newCommandStartExampleAPIServer(options, stopCh) diff --git a/pkg/cmd/grafana/apiserver/server.go b/pkg/cmd/grafana/apiserver/server.go index 28e51181154..d91cf77ac22 100644 --- a/pkg/cmd/grafana/apiserver/server.go +++ b/pkg/cmd/grafana/apiserver/server.go @@ -9,16 +9,17 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" utilerrors "k8s.io/apimachinery/pkg/util/errors" genericapiserver "k8s.io/apiserver/pkg/server" - "k8s.io/apiserver/pkg/server/options" "k8s.io/client-go/tools/clientcmd" netutils "k8s.io/utils/net" "github.com/grafana/grafana/pkg/apiserver/builder" + "github.com/grafana/grafana/pkg/infra/log" grafanaAPIServer "github.com/grafana/grafana/pkg/services/apiserver" - grafanaAPIServerOptions "github.com/grafana/grafana/pkg/services/apiserver/options" "github.com/grafana/grafana/pkg/services/apiserver/standalone" + standaloneoptions "github.com/grafana/grafana/pkg/services/apiserver/standalone/options" "github.com/grafana/grafana/pkg/services/apiserver/utils" "github.com/grafana/grafana/pkg/setting" + "github.com/spf13/pflag" ) const ( @@ -28,25 +29,24 @@ const ( // APIServerOptions contains the state for the apiserver type APIServerOptions struct { - factory standalone.APIServerFactory - builders []builder.APIGroupBuilder - ExtraOptions *grafanaAPIServerOptions.ExtraOptions - RecommendedOptions *options.RecommendedOptions - AlternateDNS []string + factory standalone.APIServerFactory + builders []builder.APIGroupBuilder + Options *standaloneoptions.Options + AlternateDNS []string + logger log.Logger StdOut io.Writer StdErr io.Writer } func newAPIServerOptions(out, errOut io.Writer) *APIServerOptions { + logger := log.New("grafana-apiserver") + return &APIServerOptions{ - StdOut: out, - StdErr: errOut, - RecommendedOptions: options.NewRecommendedOptions( - defaultEtcdPathPrefix, - grafanaAPIServer.Codecs.LegacyCodec(), // the codec is passed to etcd and not used - ), - ExtraOptions: grafanaAPIServerOptions.NewExtraOptions(), + logger: logger, + StdOut: out, + StdErr: errOut, + Options: standaloneoptions.New(logger, grafanaAPIServer.Codecs.LegacyCodec()), } } @@ -73,92 +73,31 @@ func (o *APIServerOptions) loadAPIGroupBuilders(apis []schema.GroupVersion) erro return nil } -// A copy of ApplyTo in recommended.go, but for >= 0.28, server pkg in apiserver does a bit extra causing -// a panic when CoreAPI is set to nil -func (o *APIServerOptions) ModifiedApplyTo(config *genericapiserver.RecommendedConfig) error { - if err := o.RecommendedOptions.Etcd.ApplyTo(&config.Config); err != nil { - return err - } - if err := o.RecommendedOptions.EgressSelector.ApplyTo(&config.Config); err != nil { - return err - } - if err := o.RecommendedOptions.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil { - return err - } - if err := o.RecommendedOptions.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil { - return err - } - if err := o.RecommendedOptions.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil { - return err - } - if err := o.RecommendedOptions.Authorization.ApplyTo(&config.Config.Authorization); err != nil { - return err - } - if err := o.RecommendedOptions.Audit.ApplyTo(&config.Config); err != nil { - return err - } - - // TODO: determine whether we need flow control (API priority and fairness) - // We can't assume that a shared informers config was provided in standalone mode and will need a guard - // when enabling below - /* kubeClient, err := kubernetes.NewForConfig(config.ClientConfig) - if err != nil { - return err - } - - if err := o.RecommendedOptions.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil { - return err - } */ - - if err := o.RecommendedOptions.CoreAPI.ApplyTo(config); err != nil { - return err - } - - _, err := o.RecommendedOptions.ExtraAdmissionInitializers(config) - if err != nil { - return err - } - - return nil -} - func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error) { - if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts( + if err := o.Options.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts( "localhost", o.AlternateDNS, []net.IP{netutils.ParseIPSloppy("127.0.0.1")}, ); err != nil { return nil, fmt.Errorf("error creating self-signed certificates: %v", err) } - o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true + o.Options.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true // TODO: determine authorization, currently insecure because Authorization provided by recommended options doesn't work // reason: an aggregated server won't be able to post subjectaccessreviews (Grafana doesn't have this kind) // exact error: the server could not find the requested resource (post subjectaccessreviews.authorization.k8s.io) - o.RecommendedOptions.Authorization = nil + o.Options.RecommendedOptions.Authorization = nil - o.RecommendedOptions.Admission = nil - o.RecommendedOptions.Etcd = nil + o.Options.RecommendedOptions.Admission = nil + o.Options.RecommendedOptions.Etcd = nil - if o.RecommendedOptions.CoreAPI.CoreAPIKubeconfigPath == "" { - o.RecommendedOptions.CoreAPI = nil + if o.Options.RecommendedOptions.CoreAPI.CoreAPIKubeconfigPath == "" { + o.Options.RecommendedOptions.CoreAPI = nil } serverConfig := genericapiserver.NewRecommendedConfig(grafanaAPIServer.Codecs) - if o.RecommendedOptions.CoreAPI == nil { - if err := o.ModifiedApplyTo(serverConfig); err != nil { - return nil, err - } - } else { - if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { - return nil, err - } - } - - if o.ExtraOptions != nil { - if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil { - return nil, err - } + if err := o.Options.ApplyTo(serverConfig); err != nil { + return nil, fmt.Errorf("failed to apply options to server config: %w", err) } serverConfig.DisabledPostStartHooks = serverConfig.DisabledPostStartHooks.Insert("generic-apiserver-start-informers") @@ -177,16 +116,26 @@ func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error) return serverConfig, err } +func (o *APIServerOptions) AddFlags(fs *pflag.FlagSet) { + o.Options.AddFlags(fs) + + if factoryOptions := o.factory.GetOptions(); factoryOptions != nil { + factoryOptions.AddFlags(fs) + } +} + // Validate validates APIServerOptions func (o *APIServerOptions) Validate() error { errors := make([]error, 0) - // NOTE: we don't call validate on the top level recommended options as it doesn't like skipping etcd-servers - // the function is left here for troubleshooting any other config issues - // errors = append(errors, o.RecommendedOptions.Validate()...) + if factoryOptions := o.factory.GetOptions(); factoryOptions != nil { errors = append(errors, factoryOptions.ValidateOptions()...) } + if errs := o.Options.Validate(); len(errs) > 0 { + errors = append(errors, errors...) + } + return utilerrors.NewAggregate(errors) } @@ -211,7 +160,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf } // write the local config to disk - if o.ExtraOptions.DevMode { + if o.Options.ExtraOptions.DevMode { if err = clientcmd.WriteToFile( utils.FormatKubeConfig(server.LoopbackClientConfig), path.Join(dataPath, "apiserver.kubeconfig"), diff --git a/pkg/cmd/grafana/main.go b/pkg/cmd/grafana/main.go index 696198e40b9..9985f6e64e8 100644 --- a/pkg/cmd/grafana/main.go +++ b/pkg/cmd/grafana/main.go @@ -41,7 +41,14 @@ func main() { SkipFlagParsing: true, Action: func(context *cli.Context) error { // exit here because apiserver handles its own error output - os.Exit(apiserver.RunCLI()) + os.Exit(apiserver.RunCLI(gsrv.ServerOptions{ + Version: version, + Commit: commit, + EnterpriseCommit: enterpriseCommit, + BuildBranch: buildBranch, + BuildStamp: buildstamp, + Context: context, + })) return nil }, }, diff --git a/pkg/infra/metrics/service.go b/pkg/infra/metrics/service.go index dd8d57aec5c..5a39501f82f 100644 --- a/pkg/infra/metrics/service.go +++ b/pkg/infra/metrics/service.go @@ -59,11 +59,11 @@ func (im *InternalMetricsService) Run(ctx context.Context) error { return ctx.Err() } -func ProvideRegisterer(cfg *setting.Cfg) prometheus.Registerer { +func ProvideRegisterer() prometheus.Registerer { return legacyregistry.Registerer() } -func ProvideGatherer(cfg *setting.Cfg) prometheus.Gatherer { +func ProvideGatherer() prometheus.Gatherer { k8sGatherer := newAddPrefixWrapper(legacyregistry.DefaultGatherer) return newMultiRegistry(k8sGatherer, prometheus.DefaultGatherer) } diff --git a/pkg/infra/tracing/test_helper.go b/pkg/infra/tracing/test_helper.go index b1d24abe671..81d714bb7fb 100644 --- a/pkg/infra/tracing/test_helper.go +++ b/pkg/infra/tracing/test_helper.go @@ -16,7 +16,7 @@ func WithSpanProcessor(sp tracesdk.SpanProcessor) TracerForTestOption { func InitializeTracerForTest(opts ...TracerForTestOption) Tracer { exp := tracetest.NewInMemoryExporter() - tp, _ := initTracerProvider(exp, "testing", tracesdk.AlwaysSample()) + tp, _ := initTracerProvider(exp, "grafana", "testing", tracesdk.AlwaysSample()) for _, opt := range opts { opt(tp) @@ -24,7 +24,9 @@ func InitializeTracerForTest(opts ...TracerForTestOption) Tracer { otel.SetTracerProvider(tp) - ots := &TracingService{Propagation: "jaeger,w3c", tracerProvider: tp} + cfg := NewEmptyTracingConfig() + cfg.Propagation = "jaeger,w3c" + ots := &TracingService{cfg: cfg, tracerProvider: tp} _ = ots.initOpentelemetryTracer() return ots } diff --git a/pkg/infra/tracing/tracing.go b/pkg/infra/tracing/tracing.go index cd3049555df..233e8cbd604 100644 --- a/pkg/infra/tracing/tracing.go +++ b/pkg/infra/tracing/tracing.go @@ -6,7 +6,6 @@ import ( "math" "net" "net/http" - "os" "strings" "sync" "time" @@ -28,7 +27,6 @@ import ( "github.com/go-kit/log/level" "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/setting" ) const ( @@ -46,21 +44,11 @@ const ( ) type TracingService struct { - enabled string - Address string - Propagation string - customAttribs []attribute.KeyValue - - Sampler string - SamplerParam float64 - SamplerRemoteURL string - + cfg *TracingConfig log log.Logger tracerProvider tracerProvider trace.Tracer - - Cfg *setting.Cfg } type tracerProvider interface { @@ -84,10 +72,9 @@ type Tracer interface { Inject(context.Context, http.Header, trace.Span) } -func ProvideService(cfg *setting.Cfg) (*TracingService, error) { - ots, err := ParseSettings(cfg) - if err != nil { - return nil, err +func ProvideService(tracingCfg *TracingConfig) (*TracingService, error) { + if tracingCfg == nil { + return nil, fmt.Errorf("tracingCfg cannot be nil") } log.RegisterContextualLogProvider(func(ctx context.Context) ([]any, bool) { @@ -97,21 +84,18 @@ func ProvideService(cfg *setting.Cfg) (*TracingService, error) { return nil, false }) + + ots := &TracingService{ + cfg: tracingCfg, + log: log.New("tracing"), + } + if err := ots.initOpentelemetryTracer(); err != nil { return nil, err } return ots, nil } -func ParseSettings(cfg *setting.Cfg) (*TracingService, error) { - ots := &TracingService{ - Cfg: cfg, - log: log.New("tracing"), - } - err := ots.parseSettings() - return ots, err -} - func (ots *TracingService) GetTracerProvider() tracerProvider { return ots.tracerProvider } @@ -133,96 +117,17 @@ func (noopTracerProvider) Shutdown(ctx context.Context) error { return nil } -func (ots *TracingService) parseSettings() error { - legacyAddress, legacyTags := "", "" - if section, err := ots.Cfg.Raw.GetSection("tracing.jaeger"); err == nil { - legacyAddress = section.Key("address").MustString("") - if legacyAddress == "" { - host, port := os.Getenv(envJaegerAgentHost), os.Getenv(envJaegerAgentPort) - if host != "" || port != "" { - legacyAddress = fmt.Sprintf("%s:%s", host, port) - } - } - legacyTags = section.Key("always_included_tag").MustString("") - ots.Sampler = section.Key("sampler_type").MustString("") - ots.SamplerParam = section.Key("sampler_param").MustFloat64(1) - ots.SamplerRemoteURL = section.Key("sampling_server_url").MustString("") - } - section := ots.Cfg.Raw.Section("tracing.opentelemetry") - var err error - // we default to legacy tag set (attributes) if the new config format is absent - ots.customAttribs, err = splitCustomAttribs(section.Key("custom_attributes").MustString(legacyTags)) - if err != nil { - return err - } - - // if sampler_type is set in tracing.opentelemetry, we ignore the config in tracing.jaeger - sampler := section.Key("sampler_type").MustString("") - if sampler != "" { - ots.Sampler = sampler - } - - samplerParam := section.Key("sampler_param").MustFloat64(0) - if samplerParam != 0 { - ots.SamplerParam = samplerParam - } - - samplerRemoteURL := section.Key("sampling_server_url").MustString("") - if samplerRemoteURL != "" { - ots.SamplerRemoteURL = samplerRemoteURL - } - - section = ots.Cfg.Raw.Section("tracing.opentelemetry.jaeger") - ots.enabled = noopExporter - - // we default to legacy Jaeger agent address if the new config value is empty - ots.Address = section.Key("address").MustString(legacyAddress) - ots.Propagation = section.Key("propagation").MustString("") - if ots.Address != "" { - ots.enabled = jaegerExporter - return nil - } - - section = ots.Cfg.Raw.Section("tracing.opentelemetry.otlp") - ots.Address = section.Key("address").MustString("") - if ots.Address != "" { - ots.enabled = otlpExporter - } - ots.Propagation = section.Key("propagation").MustString("") - return nil -} - -func (ots *TracingService) OTelExporterEnabled() bool { - return ots.enabled == otlpExporter -} - -func splitCustomAttribs(s string) ([]attribute.KeyValue, error) { - res := []attribute.KeyValue{} - - attribs := strings.Split(s, ",") - for _, v := range attribs { - parts := strings.SplitN(v, ":", 2) - if len(parts) > 1 { - res = append(res, attribute.String(parts[0], parts[1])) - } else if v != "" { - return nil, fmt.Errorf("custom attribute malformed - must be in 'key:value' form: %q", v) - } - } - - return res, nil -} - func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) { var ep jaeger.EndpointOption // Create the Jaeger exporter: address can be either agent address (host:port) or collector URL - if strings.HasPrefix(ots.Address, "http://") || strings.HasPrefix(ots.Address, "https://") { - ots.log.Debug("using jaeger collector", "address", ots.Address) - ep = jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.Address)) - } else if host, port, err := net.SplitHostPort(ots.Address); err == nil { + if strings.HasPrefix(ots.cfg.Address, "http://") || strings.HasPrefix(ots.cfg.Address, "https://") { + ots.log.Debug("using jaeger collector", "address", ots.cfg.Address) + ep = jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.cfg.Address)) + } else if host, port, err := net.SplitHostPort(ots.cfg.Address); err == nil { ots.log.Debug("using jaeger agent", "host", host, "port", port) ep = jaeger.WithAgentEndpoint(jaeger.WithAgentHost(host), jaeger.WithAgentPort(port), jaeger.WithMaxPacketSize(64000)) } else { - return nil, fmt.Errorf("invalid tracer address: %s", ots.Address) + return nil, fmt.Errorf("invalid tracer address: %s", ots.cfg.Address) } exp, err := jaeger.New(ep) if err != nil { @@ -234,10 +139,10 @@ func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider, resource.WithAttributes( // TODO: why are these attributes different from ones added to the // OTLP provider? - semconv.ServiceNameKey.String("grafana"), + semconv.ServiceNameKey.String(ots.cfg.ServiceName), attribute.String("environment", "production"), ), - resource.WithAttributes(ots.customAttribs...), + resource.WithAttributes(ots.cfg.CustomAttribs...), ) if err != nil { return nil, err @@ -258,7 +163,7 @@ func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider, } func (ots *TracingService) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) { - client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.Address), otlptracegrpc.WithInsecure()) + client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.cfg.Address), otlptracegrpc.WithInsecure()) exp, err := otlptrace.New(context.Background(), client) if err != nil { return nil, err @@ -269,39 +174,39 @@ func (ots *TracingService) initOTLPTracerProvider() (*tracesdk.TracerProvider, e return nil, err } - return initTracerProvider(exp, ots.Cfg.BuildVersion, sampler, ots.customAttribs...) + return initTracerProvider(exp, ots.cfg.ServiceName, ots.cfg.ServiceVersion, sampler, ots.cfg.CustomAttribs...) } func (ots *TracingService) initSampler() (tracesdk.Sampler, error) { - switch ots.Sampler { + switch ots.cfg.Sampler { case "const", "": - if ots.SamplerParam >= 1 { + if ots.cfg.SamplerParam >= 1 { return tracesdk.AlwaysSample(), nil - } else if ots.SamplerParam <= 0 { + } else if ots.cfg.SamplerParam <= 0 { return tracesdk.NeverSample(), nil } - return nil, fmt.Errorf("invalid param for const sampler - must be 0 or 1: %f", ots.SamplerParam) + return nil, fmt.Errorf("invalid param for const sampler - must be 0 or 1: %f", ots.cfg.SamplerParam) case "probabilistic": - return tracesdk.TraceIDRatioBased(ots.SamplerParam), nil + return tracesdk.TraceIDRatioBased(ots.cfg.SamplerParam), nil case "rateLimiting": - return newRateLimiter(ots.SamplerParam), nil + return newRateLimiter(ots.cfg.SamplerParam), nil case "remote": return jaegerremote.New("grafana", - jaegerremote.WithSamplingServerURL(ots.SamplerRemoteURL), - jaegerremote.WithInitialSampler(tracesdk.TraceIDRatioBased(ots.SamplerParam)), + jaegerremote.WithSamplingServerURL(ots.cfg.SamplerRemoteURL), + jaegerremote.WithInitialSampler(tracesdk.TraceIDRatioBased(ots.cfg.SamplerParam)), ), nil default: - return nil, fmt.Errorf("invalid sampler type: %s", ots.Sampler) + return nil, fmt.Errorf("invalid sampler type: %s", ots.cfg.Sampler) } } -func initTracerProvider(exp tracesdk.SpanExporter, version string, sampler tracesdk.Sampler, customAttribs ...attribute.KeyValue) (*tracesdk.TracerProvider, error) { +func initTracerProvider(exp tracesdk.SpanExporter, serviceName string, serviceVersion string, sampler tracesdk.Sampler, customAttribs ...attribute.KeyValue) (*tracesdk.TracerProvider, error) { res, err := resource.New( context.Background(), resource.WithAttributes( - semconv.ServiceNameKey.String("grafana"), - semconv.ServiceVersionKey.String(version), + semconv.ServiceNameKey.String(serviceName), + semconv.ServiceVersionKey.String(serviceVersion), ), resource.WithAttributes(customAttribs...), resource.WithProcessRuntimeDescription(), @@ -326,7 +231,7 @@ func (ots *TracingService) initNoopTracerProvider() (tracerProvider, error) { func (ots *TracingService) initOpentelemetryTracer() error { var tp tracerProvider var err error - switch ots.enabled { + switch ots.cfg.enabled { case jaegerExporter: tp, err = ots.initJaegerTracerProvider() if err != nil { @@ -347,12 +252,12 @@ func (ots *TracingService) initOpentelemetryTracer() error { // Register our TracerProvider as the global so any imported // instrumentation in the future will default to using it // only if tracing is enabled - if ots.enabled != "" { + if ots.cfg.enabled != "" { otel.SetTracerProvider(tp) } propagators := []propagation.TextMapPropagator{} - for _, p := range strings.Split(ots.Propagation, ",") { + for _, p := range strings.Split(ots.cfg.Propagation, ",") { switch p { case w3cPropagator: propagators = append(propagators, propagation.TraceContext{}, propagation.Baggage{}) diff --git a/pkg/infra/tracing/tracing_config.go b/pkg/infra/tracing/tracing_config.go new file mode 100644 index 00000000000..14c878a2351 --- /dev/null +++ b/pkg/infra/tracing/tracing_config.go @@ -0,0 +1,144 @@ +package tracing + +import ( + "fmt" + "os" + "strings" + + "github.com/grafana/grafana/pkg/setting" + "go.opentelemetry.io/otel/attribute" +) + +type TracingConfig struct { + enabled string + Address string + Propagation string + CustomAttribs []attribute.KeyValue + + Sampler string + SamplerParam float64 + SamplerRemoteURL string + + ServiceName string + ServiceVersion string +} + +func ProvideTracingConfig(cfg *setting.Cfg) (*TracingConfig, error) { + return ParseTracingConfig(cfg) +} + +func NewEmptyTracingConfig() *TracingConfig { + return &TracingConfig{ + CustomAttribs: []attribute.KeyValue{}, + } +} + +func NewJaegerTracingConfig(address string, propagation string) (*TracingConfig, error) { + if address == "" { + return nil, fmt.Errorf("address cannot be empty") + } + + cfg := NewEmptyTracingConfig() + cfg.enabled = jaegerExporter + cfg.Address = address + cfg.Propagation = propagation + return cfg, nil +} + +func NewOTLPTracingConfig(address string, propagation string) (*TracingConfig, error) { + if address == "" { + return nil, fmt.Errorf("address cannot be empty") + } + + cfg := NewEmptyTracingConfig() + cfg.enabled = otlpExporter + cfg.Address = address + cfg.Propagation = propagation + return cfg, nil +} + +func ParseTracingConfig(cfg *setting.Cfg) (*TracingConfig, error) { + if cfg == nil { + return nil, fmt.Errorf("cfg cannot be nil") + } + tc := NewEmptyTracingConfig() + tc.ServiceName = "grafana" + tc.ServiceVersion = cfg.BuildVersion + + legacyAddress, legacyTags := "", "" + if section, err := cfg.Raw.GetSection("tracing.jaeger"); err == nil { + legacyAddress = section.Key("address").MustString("") + if legacyAddress == "" { + host, port := os.Getenv(envJaegerAgentHost), os.Getenv(envJaegerAgentPort) + if host != "" || port != "" { + legacyAddress = fmt.Sprintf("%s:%s", host, port) + } + } + legacyTags = section.Key("always_included_tag").MustString("") + tc.Sampler = section.Key("sampler_type").MustString("") + tc.SamplerParam = section.Key("sampler_param").MustFloat64(1) + tc.SamplerRemoteURL = section.Key("sampling_server_url").MustString("") + } + section := cfg.Raw.Section("tracing.opentelemetry") + var err error + // we default to legacy tag set (attributes) if the new config format is absent + tc.CustomAttribs, err = splitCustomAttribs(section.Key("custom_attributes").MustString(legacyTags)) + if err != nil { + return nil, err + } + + // if sampler_type is set in tracing.opentelemetry, we ignore the config in tracing.jaeger + sampler := section.Key("sampler_type").MustString("") + if sampler != "" { + tc.Sampler = sampler + } + + samplerParam := section.Key("sampler_param").MustFloat64(0) + if samplerParam != 0 { + tc.SamplerParam = samplerParam + } + + samplerRemoteURL := section.Key("sampling_server_url").MustString("") + if samplerRemoteURL != "" { + tc.SamplerRemoteURL = samplerRemoteURL + } + + section = cfg.Raw.Section("tracing.opentelemetry.jaeger") + tc.enabled = noopExporter + + // we default to legacy Jaeger agent address if the new config value is empty + tc.Address = section.Key("address").MustString(legacyAddress) + tc.Propagation = section.Key("propagation").MustString("") + if tc.Address != "" { + tc.enabled = jaegerExporter + return tc, nil + } + + section = cfg.Raw.Section("tracing.opentelemetry.otlp") + tc.Address = section.Key("address").MustString("") + if tc.Address != "" { + tc.enabled = otlpExporter + } + tc.Propagation = section.Key("propagation").MustString("") + return tc, nil +} + +func (tc TracingConfig) OTelExporterEnabled() bool { + return tc.enabled == otlpExporter +} + +func splitCustomAttribs(s string) ([]attribute.KeyValue, error) { + res := []attribute.KeyValue{} + + attribs := strings.Split(s, ",") + for _, v := range attribs { + parts := strings.SplitN(v, ":", 2) + if len(parts) > 1 { + res = append(res, attribute.String(parts[0], parts[1])) + } else if v != "" { + return nil, fmt.Errorf("custom attribute malformed - must be in 'key:value' form: %q", v) + } + } + + return res, nil +} diff --git a/pkg/infra/tracing/tracing_config_test.go b/pkg/infra/tracing/tracing_config_test.go new file mode 100644 index 00000000000..e8d6be3e109 --- /dev/null +++ b/pkg/infra/tracing/tracing_config_test.go @@ -0,0 +1,190 @@ +package tracing + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "go.opentelemetry.io/otel/attribute" + + "github.com/grafana/grafana/pkg/setting" +) + +// TODO(zserge) Add proper tests for opentelemetry + +func TestSplitCustomAttribs(t *testing.T) { + tests := []struct { + input string + expected []attribute.KeyValue + }{ + { + input: "key1:value:1", + expected: []attribute.KeyValue{attribute.String("key1", "value:1")}, + }, + { + input: "key1:value1,key2:value2", + expected: []attribute.KeyValue{ + attribute.String("key1", "value1"), + attribute.String("key2", "value2"), + }, + }, + { + input: "", + expected: []attribute.KeyValue{}, + }, + } + + for _, test := range tests { + attribs, err := splitCustomAttribs(test.input) + assert.NoError(t, err) + assert.EqualValues(t, test.expected, attribs) + } +} + +func TestSplitCustomAttribs_Malformed(t *testing.T) { + tests := []struct { + input string + }{ + {input: "key1=value1"}, + {input: "key1"}, + } + + for _, test := range tests { + _, err := splitCustomAttribs(test.input) + assert.Error(t, err) + } +} + +func TestTracingConfig(t *testing.T) { + for _, test := range []struct { + Name string + Cfg string + Env map[string]string + ExpectedExporter string + ExpectedAddress string + ExpectedPropagator string + ExpectedAttrs []attribute.KeyValue + + ExpectedSampler string + ExpectedSamplerParam float64 + ExpectedSamplingServerURL string + }{ + { + Name: "default config uses noop exporter", + Cfg: "", + ExpectedExporter: noopExporter, + ExpectedAttrs: []attribute.KeyValue{}, + }, + { + Name: "custom attributes are parsed", + Cfg: ` + [tracing.opentelemetry] + custom_attributes = key1:value1,key2:value2 + `, + ExpectedExporter: noopExporter, + ExpectedAttrs: []attribute.KeyValue{attribute.String("key1", "value1"), attribute.String("key2", "value2")}, + }, + { + Name: "jaeger address is parsed", + Cfg: ` + [tracing.opentelemetry.jaeger] + address = jaeger.example.com:6831 + `, + ExpectedExporter: jaegerExporter, + ExpectedAddress: "jaeger.example.com:6831", + ExpectedAttrs: []attribute.KeyValue{}, + }, + { + Name: "OTLP address is parsed", + Cfg: ` + [tracing.opentelemetry.otlp] + address = otlp.example.com:4317 + `, + ExpectedExporter: otlpExporter, + ExpectedAddress: "otlp.example.com:4317", + ExpectedAttrs: []attribute.KeyValue{}, + }, + { + Name: "legacy config format is supported", + Cfg: ` + [tracing.jaeger] + address = jaeger.example.com:6831 + `, + ExpectedExporter: jaegerExporter, + ExpectedAddress: "jaeger.example.com:6831", + ExpectedAttrs: []attribute.KeyValue{}, + }, + { + Name: "legacy env variables are supported", + Cfg: `[tracing.jaeger]`, + Env: map[string]string{ + "JAEGER_AGENT_HOST": "example.com", + "JAEGER_AGENT_PORT": "12345", + }, + ExpectedExporter: jaegerExporter, + ExpectedAddress: "example.com:12345", + ExpectedAttrs: []attribute.KeyValue{}, + }, + { + Name: "opentelemetry config format is prioritised over legacy jaeger", + Cfg: ` + [tracing.jaeger] + address = foo.com:6831 + custom_tags = a:b + sampler_param = 0 + [tracing.opentelemetry] + custom_attributes = c:d + sampler_param = 1 + [tracing.opentelemetry.jaeger] + address = bar.com:6831 + `, + ExpectedExporter: jaegerExporter, + ExpectedAddress: "bar.com:6831", + ExpectedAttrs: []attribute.KeyValue{attribute.String("c", "d")}, + ExpectedSamplerParam: 1.0, + }, + { + Name: "remote sampler config is parsed from otel config", + Cfg: ` + [tracing.opentelemetry] + sampler_type = remote + sampler_param = 0.5 + sampling_server_url = http://example.com:5778/sampling + [tracing.opentelemetry.otlp] + address = otlp.example.com:4317 + `, + ExpectedExporter: otlpExporter, + ExpectedAddress: "otlp.example.com:4317", + ExpectedAttrs: []attribute.KeyValue{}, + ExpectedSampler: "remote", + ExpectedSamplerParam: 0.5, + ExpectedSamplingServerURL: "http://example.com:5778/sampling", + }, + } { + t.Run(test.Name, func(t *testing.T) { + // export environment variables + if test.Env != nil { + for k, v := range test.Env { + t.Setenv(k, v) + } + } + // parse config sections + cfg := setting.NewCfg() + err := cfg.Raw.Append([]byte(test.Cfg)) + assert.NoError(t, err) + // create tracingConfig + tracingConfig, err := ProvideTracingConfig(cfg) + assert.NoError(t, err) + // make sure tracker is properly configured + assert.Equal(t, test.ExpectedExporter, tracingConfig.enabled) + assert.Equal(t, test.ExpectedAddress, tracingConfig.Address) + assert.Equal(t, test.ExpectedPropagator, tracingConfig.Propagation) + assert.Equal(t, test.ExpectedAttrs, tracingConfig.CustomAttribs) + + if test.ExpectedSampler != "" { + assert.Equal(t, test.ExpectedSampler, tracingConfig.Sampler) + assert.Equal(t, test.ExpectedSamplerParam, tracingConfig.SamplerParam) + assert.Equal(t, test.ExpectedSamplingServerURL, tracingConfig.SamplerRemoteURL) + } + }) + } +} diff --git a/pkg/infra/tracing/tracing_test.go b/pkg/infra/tracing/tracing_test.go index 53a56d897a1..b52b772d424 100644 --- a/pkg/infra/tracing/tracing_test.go +++ b/pkg/infra/tracing/tracing_test.go @@ -5,220 +5,38 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.opentelemetry.io/otel/attribute" - - "github.com/grafana/grafana/pkg/setting" ) -// TODO(zserge) Add proper tests for opentelemetry - -func TestSplitCustomAttribs(t *testing.T) { - tests := []struct { - input string - expected []attribute.KeyValue - }{ - { - input: "key1:value:1", - expected: []attribute.KeyValue{attribute.String("key1", "value:1")}, - }, - { - input: "key1:value1,key2:value2", - expected: []attribute.KeyValue{ - attribute.String("key1", "value1"), - attribute.String("key2", "value2"), - }, - }, - { - input: "", - expected: []attribute.KeyValue{}, - }, - } - - for _, test := range tests { - attribs, err := splitCustomAttribs(test.input) - assert.NoError(t, err) - assert.EqualValues(t, test.expected, attribs) - } -} - -func TestSplitCustomAttribs_Malformed(t *testing.T) { - tests := []struct { - input string - }{ - {input: "key1=value1"}, - {input: "key1"}, - } - - for _, test := range tests { - _, err := splitCustomAttribs(test.input) - assert.Error(t, err) - } -} - -func TestTracingConfig(t *testing.T) { - for _, test := range []struct { - Name string - Cfg string - Env map[string]string - ExpectedExporter string - ExpectedAddress string - ExpectedPropagator string - ExpectedAttrs []attribute.KeyValue - - ExpectedSampler string - ExpectedSamplerParam float64 - ExpectedSamplingServerURL string - }{ - { - Name: "default config uses noop exporter", - Cfg: "", - ExpectedExporter: noopExporter, - ExpectedAttrs: []attribute.KeyValue{}, - }, - { - Name: "custom attributes are parsed", - Cfg: ` - [tracing.opentelemetry] - custom_attributes = key1:value1,key2:value2 - `, - ExpectedExporter: noopExporter, - ExpectedAttrs: []attribute.KeyValue{attribute.String("key1", "value1"), attribute.String("key2", "value2")}, - }, - { - Name: "jaeger address is parsed", - Cfg: ` - [tracing.opentelemetry.jaeger] - address = jaeger.example.com:6831 - `, - ExpectedExporter: jaegerExporter, - ExpectedAddress: "jaeger.example.com:6831", - ExpectedAttrs: []attribute.KeyValue{}, - }, - { - Name: "OTLP address is parsed", - Cfg: ` - [tracing.opentelemetry.otlp] - address = otlp.example.com:4317 - `, - ExpectedExporter: otlpExporter, - ExpectedAddress: "otlp.example.com:4317", - ExpectedAttrs: []attribute.KeyValue{}, - }, - { - Name: "legacy config format is supported", - Cfg: ` - [tracing.jaeger] - address = jaeger.example.com:6831 - `, - ExpectedExporter: jaegerExporter, - ExpectedAddress: "jaeger.example.com:6831", - ExpectedAttrs: []attribute.KeyValue{}, - }, - { - Name: "legacy env variables are supported", - Cfg: `[tracing.jaeger]`, - Env: map[string]string{ - "JAEGER_AGENT_HOST": "example.com", - "JAEGER_AGENT_PORT": "12345", - }, - ExpectedExporter: jaegerExporter, - ExpectedAddress: "example.com:12345", - ExpectedAttrs: []attribute.KeyValue{}, - }, - { - Name: "opentelemetry config format is prioritised over legacy jaeger", - Cfg: ` - [tracing.jaeger] - address = foo.com:6831 - custom_tags = a:b - sampler_param = 0 - [tracing.opentelemetry] - custom_attributes = c:d - sampler_param = 1 - [tracing.opentelemetry.jaeger] - address = bar.com:6831 - `, - ExpectedExporter: jaegerExporter, - ExpectedAddress: "bar.com:6831", - ExpectedAttrs: []attribute.KeyValue{attribute.String("c", "d")}, - ExpectedSamplerParam: 1.0, - }, - { - Name: "remote sampler config is parsed from otel config", - Cfg: ` - [tracing.opentelemetry] - sampler_type = remote - sampler_param = 0.5 - sampling_server_url = http://example.com:5778/sampling - [tracing.opentelemetry.otlp] - address = otlp.example.com:4317 - `, - ExpectedExporter: otlpExporter, - ExpectedAddress: "otlp.example.com:4317", - ExpectedAttrs: []attribute.KeyValue{}, - ExpectedSampler: "remote", - ExpectedSamplerParam: 0.5, - ExpectedSamplingServerURL: "http://example.com:5778/sampling", - }, - } { - t.Run(test.Name, func(t *testing.T) { - // export environment variables - if test.Env != nil { - for k, v := range test.Env { - t.Setenv(k, v) - } - } - // parse config sections - cfg := setting.NewCfg() - err := cfg.Raw.Append([]byte(test.Cfg)) - assert.NoError(t, err) - // create tracer - tracer, err := ProvideService(cfg) - assert.NoError(t, err) - // make sure tracker is properly configured - assert.Equal(t, test.ExpectedExporter, tracer.enabled) - assert.Equal(t, test.ExpectedAddress, tracer.Address) - assert.Equal(t, test.ExpectedPropagator, tracer.Propagation) - assert.Equal(t, test.ExpectedAttrs, tracer.customAttribs) - - if test.ExpectedSampler != "" { - assert.Equal(t, test.ExpectedSampler, tracer.Sampler) - assert.Equal(t, test.ExpectedSamplerParam, tracer.SamplerParam) - assert.Equal(t, test.ExpectedSamplingServerURL, tracer.SamplerRemoteURL) - } - }) - } -} - func TestInitSampler(t *testing.T) { otel := &TracingService{} + otel.cfg = NewEmptyTracingConfig() sampler, err := otel.initSampler() require.NoError(t, err) assert.Equal(t, "AlwaysOffSampler", sampler.Description()) - otel.Sampler = "bogus" + otel.cfg.Sampler = "bogus" _, err = otel.initSampler() require.Error(t, err) - otel.Sampler = "const" - otel.SamplerParam = 0.5 + otel.cfg.Sampler = "const" + otel.cfg.SamplerParam = 0.5 _, err = otel.initSampler() require.Error(t, err) - otel.Sampler = "const" - otel.SamplerParam = 1.0 + otel.cfg.Sampler = "const" + otel.cfg.SamplerParam = 1.0 sampler, err = otel.initSampler() require.NoError(t, err) assert.Equal(t, "AlwaysOnSampler", sampler.Description()) - otel.Sampler = "probabilistic" - otel.SamplerParam = 0.5 + otel.cfg.Sampler = "probabilistic" + otel.cfg.SamplerParam = 0.5 sampler, err = otel.initSampler() require.NoError(t, err) assert.Equal(t, "TraceIDRatioBased{0.5}", sampler.Description()) - otel.Sampler = "rateLimiting" - otel.SamplerParam = 100.25 + otel.cfg.Sampler = "rateLimiting" + otel.cfg.SamplerParam = 100.25 sampler, err = otel.initSampler() require.NoError(t, err) assert.Equal(t, "RateLimitingSampler{100.25}", sampler.Description()) diff --git a/pkg/server/wire.go b/pkg/server/wire.go index e43937d8c76..64d94f62d68 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -247,6 +247,7 @@ var wireBasicSet = wire.NewSet( notifications.ProvideService, notifications.ProvideSmtpService, tracing.ProvideService, + tracing.ProvideTracingConfig, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), testdatasource.ProvideService, ldapapi.ProvideService, diff --git a/pkg/services/apiserver/options/options.go b/pkg/services/apiserver/options/options.go index 42cb6557036..3aef73d5782 100644 --- a/pkg/services/apiserver/options/options.go +++ b/pkg/services/apiserver/options/options.go @@ -27,13 +27,10 @@ type Options struct { func NewOptions(codec runtime.Codec) *Options { return &Options{ - RecommendedOptions: genericoptions.NewRecommendedOptions( - defaultEtcdPathPrefix, - codec, - ), - AggregatorOptions: NewAggregatorServerOptions(), - StorageOptions: NewStorageOptions(), - ExtraOptions: NewExtraOptions(), + RecommendedOptions: NewRecommendedOptions(codec), + AggregatorOptions: NewAggregatorServerOptions(), + StorageOptions: NewStorageOptions(), + ExtraOptions: NewExtraOptions(), } } @@ -118,6 +115,13 @@ func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) erro return nil } +func NewRecommendedOptions(codec runtime.Codec) *genericoptions.RecommendedOptions { + return genericoptions.NewRecommendedOptions( + defaultEtcdPathPrefix, + codec, + ) +} + type fakeListener struct { server net.Conn client net.Conn diff --git a/pkg/services/apiserver/standalone/options/logging.go b/pkg/services/apiserver/standalone/options/logging.go new file mode 100644 index 00000000000..4f6bf0ebb4c --- /dev/null +++ b/pkg/services/apiserver/standalone/options/logging.go @@ -0,0 +1,41 @@ +package options + +import ( + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/setting" + "github.com/spf13/pflag" + genericapiserver "k8s.io/apiserver/pkg/server" +) + +type LoggingOptions struct { + logger log.Logger + Level string +} + +func NewLoggingOptions(logger log.Logger) *LoggingOptions { + return &LoggingOptions{ + logger: logger, + } +} + +func (o *LoggingOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.Level, "grafana.log.level", "debug", "Log level, debug, info, warn, error.") +} + +func (o *LoggingOptions) Validate() []error { + return nil +} + +func (o *LoggingOptions) ApplyTo(c *genericapiserver.RecommendedConfig) error { + err := log.SetupConsoleLogger(o.Level) + if err != nil { + return err + } + + o.logger.Info("Starting grafana-apiserver", "version", setting.BuildVersion, "commit", setting.BuildCommit, "branch", setting.BuildBranch, "compiled", time.Unix(setting.BuildStamp, 0)) + o.logger.Debug("Console logging initialized", "logLevel", o.Level) + + return nil +} diff --git a/pkg/services/apiserver/standalone/options/metrics.go b/pkg/services/apiserver/standalone/options/metrics.go new file mode 100644 index 00000000000..4a213bd4606 --- /dev/null +++ b/pkg/services/apiserver/standalone/options/metrics.go @@ -0,0 +1,42 @@ +package options + +import ( + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/metrics" + "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/pflag" + genericapiserver "k8s.io/apiserver/pkg/server" +) + +type MetricsOptions struct { + logger log.Logger + Enabled bool + MetricsRegisterer prometheus.Registerer +} + +func NewMetrcicsOptions(logger log.Logger) *MetricsOptions { + return &MetricsOptions{ + logger: logger, + } +} + +func (o *MetricsOptions) AddFlags(fs *pflag.FlagSet) { + fs.BoolVar(&o.Enabled, "grafana.metrics.enable", false, "Enable metrics and Prometheus /metrics endpoint.") +} + +func (o *MetricsOptions) Validate() []error { + return nil +} + +func (o *MetricsOptions) ApplyTo(c *genericapiserver.RecommendedConfig) error { + c.EnableMetrics = o.Enabled + o.MetricsRegisterer = metrics.ProvideRegisterer() + metrics.SetBuildInformation(o.MetricsRegisterer, setting.BuildVersion, setting.BuildCommit, setting.BuildBranch, setting.BuildStamp) + + if o.Enabled { + o.logger.Debug("Metrics enabled") + } + + return nil +} diff --git a/pkg/services/apiserver/standalone/options/options.go b/pkg/services/apiserver/standalone/options/options.go new file mode 100644 index 00000000000..696b4fe4956 --- /dev/null +++ b/pkg/services/apiserver/standalone/options/options.go @@ -0,0 +1,159 @@ +package options + +import ( + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/apiserver/options" + "github.com/spf13/pflag" + "k8s.io/apimachinery/pkg/runtime" + genericapiserver "k8s.io/apiserver/pkg/server" + genericoptions "k8s.io/apiserver/pkg/server/options" +) + +type Options struct { + LoggingOptions *LoggingOptions + ExtraOptions *options.ExtraOptions + RecommendedOptions *genericoptions.RecommendedOptions + TracingOptions *TracingOptions + MetricsOptions *MetricsOptions +} + +func New(logger log.Logger, codec runtime.Codec) *Options { + return &Options{ + LoggingOptions: NewLoggingOptions(logger), + ExtraOptions: options.NewExtraOptions(), + RecommendedOptions: options.NewRecommendedOptions(codec), + TracingOptions: NewTracingOptions(logger), + MetricsOptions: NewMetrcicsOptions(logger), + } +} + +func (o *Options) AddFlags(fs *pflag.FlagSet) { + o.LoggingOptions.AddFlags(fs) + o.ExtraOptions.AddFlags(fs) + o.RecommendedOptions.AddFlags(fs) + o.TracingOptions.AddFlags(fs) + o.MetricsOptions.AddFlags(fs) +} + +func (o *Options) Validate() []error { + if errs := o.LoggingOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.ExtraOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.TracingOptions.Validate(); len(errs) != 0 { + return errs + } + + if errs := o.MetricsOptions.Validate(); len(errs) != 0 { + return errs + } + + // NOTE: we don't call validate on the top level recommended options as it doesn't like skipping etcd-servers + // the function is left here for troubleshooting any other config issues + // errors = append(errors, o.RecommendedOptions.Validate()...) + + if errs := o.RecommendedOptions.SecureServing.Validate(); len(errs) != 0 { + return errs + } + + if o.ExtraOptions.DevMode { + // NOTE: Only consider authn for dev mode - resolves the failure due to missing extension apiserver auth-config + // in parent k8s + if errs := o.RecommendedOptions.Authentication.Validate(); len(errs) != 0 { + return errs + } + } + + return nil +} + +// A copy of ApplyTo in recommended.go, but for >= 0.28, server pkg in apiserver does a bit extra causing +// a panic when CoreAPI is set to nil +func (o *Options) ModifiedApplyTo(config *genericapiserver.RecommendedConfig) error { + if err := o.RecommendedOptions.Etcd.ApplyTo(&config.Config); err != nil { + return err + } + if err := o.RecommendedOptions.EgressSelector.ApplyTo(&config.Config); err != nil { + return err + } + if err := o.RecommendedOptions.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil { + return err + } + if err := o.RecommendedOptions.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil { + return err + } + if err := o.RecommendedOptions.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil { + return err + } + if err := o.RecommendedOptions.Authorization.ApplyTo(&config.Config.Authorization); err != nil { + return err + } + if err := o.RecommendedOptions.Audit.ApplyTo(&config.Config); err != nil { + return err + } + + // TODO: determine whether we need flow control (API priority and fairness) + // We can't assume that a shared informers config was provided in standalone mode and will need a guard + // when enabling below + /* kubeClient, err := kubernetes.NewForConfig(config.ClientConfig) + if err != nil { + return err + } + + if err := o.RecommendedOptions.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil { + return err + } */ + + if err := o.RecommendedOptions.CoreAPI.ApplyTo(config); err != nil { + return err + } + + _, err := o.RecommendedOptions.ExtraAdmissionInitializers(config) + if err != nil { + return err + } + + return nil +} + +func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) error { + if o.LoggingOptions != nil { + if err := o.LoggingOptions.ApplyTo(serverConfig); err != nil { + return err + } + } + + if o.ExtraOptions != nil { + if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil { + return err + } + } + + if o.RecommendedOptions.CoreAPI == nil { + if err := o.ModifiedApplyTo(serverConfig); err != nil { + return err + } + } else { + if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil { + return err + } + } + + if o.TracingOptions != nil { + if err := o.TracingOptions.ApplyTo(serverConfig); err != nil { + return err + } + } + + if o.MetricsOptions != nil { + if err := o.MetricsOptions.ApplyTo(serverConfig); err != nil { + return err + } + } + + return nil +} diff --git a/pkg/services/apiserver/standalone/options/tracing.go b/pkg/services/apiserver/standalone/options/tracing.go new file mode 100644 index 00000000000..a330c7aec26 --- /dev/null +++ b/pkg/services/apiserver/standalone/options/tracing.go @@ -0,0 +1,124 @@ +package options + +import ( + "context" + "errors" + "fmt" + "net/url" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/tracing" + "github.com/grafana/grafana/pkg/setting" + "github.com/spf13/pflag" + "go.opentelemetry.io/otel/attribute" + genericapiserver "k8s.io/apiserver/pkg/server" +) + +type TracingOptions struct { + logger log.Logger + + JaegerAddress string + JaegerPropagation string + + OTLPAddress string + OTLPPropagation string + + Tags map[string]string + + SamplerType string + SamplerParam float64 + SamplingServiceURL string + + TracingService *tracing.TracingService +} + +func NewTracingOptions(logger log.Logger) *TracingOptions { + return &TracingOptions{ + logger: logger, + Tags: map[string]string{}, + } +} + +func (o *TracingOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVar(&o.JaegerAddress, "grafana.tracing.jaeger.address", "", "Tracing Jaeger exporter destination, e.g. http://localhost:14268/api/traces. This enabled the Jaeger export and takes presedence over grafana.tracing.otlp.") + fs.StringVar(&o.JaegerPropagation, "grafana.tracing.jaeger.propagation", "jaeger", "Tracing Jaeger propagation specifies the text map propagation format, w3c or jaeger.") + fs.StringVar(&o.OTLPAddress, "grafana.tracing.otlp.address", "", "Tracing OTLP exporter destination, e.g. localhost:4317.") + fs.StringVar(&o.OTLPPropagation, "grafana.tracing.otlp.propagation", "w3c", "Tracing OTLP propagation specifies the text map propagation format, w3c or jaeger.") + fs.StringToStringVar(&o.Tags, "grafana.tracing.tag", map[string]string{}, "Tracing server tag in 'key=value' format. Specify multiple times to add many.") + fs.StringVar(&o.SamplerType, "grafana.tracing.sampler-type", "const", "Tracing sampler type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote.") + fs.Float64Var(&o.SamplerParam, "grafana.tracing.sampler-param", 0, "Tracing sampler configuration parameter. For 'const' sampler, 0 or 1 for always false/true respectively. For 'rateLimiting' sampler, the number of spans per second. For 'remote' sampler, param is the same as for 'probabilistic' and indicates the initial sampling rate before the actual one is received from the sampling service.") + fs.StringVar(&o.SamplingServiceURL, "grafana.tracing.sampling-service", "", "Tracing server sampling service URL (used for both Jaeger and OTLP) if grafana.tracing.sampler-type=remote.") +} + +func (o *TracingOptions) Validate() []error { + errors := []error{} + + if o.JaegerAddress != "" { + if _, err := url.Parse(o.JaegerAddress); err != nil { + errors = append(errors, fmt.Errorf("failed to parse tracing.jaeger.address: %w", err)) + } + } + + if o.SamplingServiceURL != "" { + if _, err := url.Parse(o.SamplingServiceURL); err != nil { + errors = append(errors, fmt.Errorf("failed to parse tracing.sampling-service: %w", err)) + } + } + + return errors +} + +func (o *TracingOptions) ApplyTo(config *genericapiserver.RecommendedConfig) error { + tracingCfg := tracing.NewEmptyTracingConfig() + var err error + + if o.OTLPAddress != "" { + tracingCfg, err = tracing.NewOTLPTracingConfig(o.OTLPAddress, o.OTLPPropagation) + } + + if o.JaegerAddress != "" { + tracingCfg, err = tracing.NewJaegerTracingConfig(o.JaegerAddress, o.JaegerPropagation) + } + + if err != nil { + return err + } + + tracingCfg.ServiceName = "grafana-apiserver" + tracingCfg.ServiceVersion = setting.BuildVersion + + for k, v := range o.Tags { + tracingCfg.CustomAttribs = append(tracingCfg.CustomAttribs, attribute.String(k, v)) + } + + tracingCfg.Sampler = o.SamplerType + tracingCfg.SamplerParam = o.SamplerParam + tracingCfg.SamplerRemoteURL = o.SamplingServiceURL + + ts, err := tracing.ProvideService(tracingCfg) + if err != nil { + return err + } + + o.TracingService = ts + config.TracerProvider = ts.GetTracerProvider() + + config.AddPostStartHookOrDie("grafana-tracing-service", func(hookCtx genericapiserver.PostStartHookContext) error { + ctx, cancel := context.WithCancel(context.Background()) + + go func() { + <-hookCtx.StopCh + cancel() + }() + + go func() { + if err := ts.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { + o.logger.Error("failed to shutdown tracing service", "error", err) + } + }() + + return nil + }) + + return nil +} diff --git a/pkg/services/pluginsintegration/pluginconfig/tracing.go b/pkg/services/pluginsintegration/pluginconfig/tracing.go index 0eada776ceb..025cd5318eb 100644 --- a/pkg/services/pluginsintegration/pluginconfig/tracing.go +++ b/pkg/services/pluginsintegration/pluginconfig/tracing.go @@ -11,20 +11,20 @@ import ( // newTracingCfg creates a plugins tracing configuration based on the provided Grafana tracing config. // If OpenTelemetry (OTLP) is disabled, a zero-value OpenTelemetryCfg is returned. func newTracingCfg(grafanaCfg *setting.Cfg) (pCfg.Tracing, error) { - ots, err := tracing.ParseSettings(grafanaCfg) + tracingCfg, err := tracing.ParseTracingConfig(grafanaCfg) if err != nil { return pCfg.Tracing{}, fmt.Errorf("parse settings: %w", err) } - if !ots.OTelExporterEnabled() { + if !tracingCfg.OTelExporterEnabled() { return pCfg.Tracing{}, nil } return pCfg.Tracing{ OpenTelemetry: pCfg.OpenTelemetryCfg{ - Address: ots.Address, - Propagation: ots.Propagation, - Sampler: ots.Sampler, - SamplerParam: ots.SamplerParam, - SamplerRemoteURL: ots.SamplerRemoteURL, + Address: tracingCfg.Address, + Propagation: tracingCfg.Propagation, + Sampler: tracingCfg.Sampler, + SamplerParam: tracingCfg.SamplerParam, + SamplerRemoteURL: tracingCfg.SamplerRemoteURL, }, }, nil } diff --git a/pkg/services/store/entity/server/service.go b/pkg/services/store/entity/server/service.go index 1f08551cfad..f4e4468e40a 100644 --- a/pkg/services/store/entity/server/service.go +++ b/pkg/services/store/entity/server/service.go @@ -57,7 +57,12 @@ func ProvideService( cfg *setting.Cfg, features featuremgmt.FeatureToggles, ) (*service, error) { - tracing, err := tracing.ProvideService(cfg) + tracingCfg, err := tracing.ProvideTracingConfig(cfg) + if err != nil { + return nil, err + } + + tracing, err := tracing.ProvideService(tracingCfg) if err != nil { return nil, err }