IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-12-25 16:31:28 -06:00
Code Issues Packages Projects Releases Wiki Activity

API Server: Standalone observability (#84789)

Browse Source 
Adds support for logs (specify level), metrics (enable metrics and Prometheus /metrics endpoint 
and traces (jaeger or otlp) for standalone API server. This will allow any grafana core service 
part of standalone apiserver to use logging, metrics and traces as normal.
This commit is contained in:
Marcus Efraimsson 2024-03-21 17:06:32 +01:00 committed by GitHub
parent 856ed64aac
commit 6c1de260a2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 860 additions and 445 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/cmd/grafana-cli/commands/conflict_user_command.go
View File

@ -79,7 +79,12 @@ func initializeConflictResolver(cmd *utils.ContextCommandLine, f Formatter, ctx
}
func getSqlStore(cfg *setting.Cfg, features featuremgmt.FeatureToggles) (*sqlstore.SQLStore, error) {
tracer, err := tracing.ProvideService(cfg)
tracingCfg, err := tracing.ProvideTracingConfig(cfg)
if err != nil {
return nil, fmt.Errorf("%v: %w", "failed to initialize tracer config", err)
}
tracer, err := tracing.ProvideService(tracingCfg)
if err != nil {
return nil, fmt.Errorf("%v: %w", "failed to initialize tracer service", err)
}

2
pkg/cmd/grafana-server/commands/buildinfo.go
View File

@ -16,7 +16,7 @@ func getBuildstamp(opts ServerOptions) int64 {
return buildstampInt64
}
func setBuildInfo(opts ServerOptions) {
func SetBuildInfo(opts ServerOptions) {
setting.BuildVersion = opts.Version
setting.BuildCommit = opts.Commit
setting.EnterpriseBuildCommit = opts.EnterpriseCommit

4
pkg/cmd/grafana-server/commands/cli.go
View File

@ -98,7 +98,7 @@ func RunServer(opts ServerOptions) error {
}
}()
setBuildInfo(opts)
SetBuildInfo(opts)
checkPrivileges()
configOptions := strings.Split(ConfigOverrides, " ")
@ -112,7 +112,7 @@ func RunServer(opts ServerOptions) error {
return err
}
metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
metrics.SetBuildInformation(metrics.ProvideRegisterer(), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
s, err := server.Initialize(
cfg,

4
pkg/cmd/grafana-server/commands/target.go
View File

@ -75,7 +75,7 @@ func RunTargetServer(opts ServerOptions) error {
}
}()
setBuildInfo(opts)
SetBuildInfo(opts)
checkPrivileges()
configOptions := strings.Split(ConfigOverrides, " ")
@ -89,7 +89,7 @@ func RunTargetServer(opts ServerOptions) error {
return err
}
metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
metrics.SetBuildInformation(metrics.ProvideRegisterer(), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
s, err := server.InitializeModuleServer(
cfg,

24
pkg/cmd/grafana/apiserver/apiserver.md
View File

@ -1,6 +1,6 @@
# grafana apiserver (standalone)
The example-apiserver closely resembles the
The example-apiserver closely resembles the
[sample-apiserver](https://github.com/kubernetes/sample-apiserver/tree/master) project in code and thus
allows the same
[CLI flags](https://kubernetes.io/docs/reference/command-line-tools-reference/kube-apiserver/) as kube-apiserver.
@ -32,3 +32,25 @@ dummy example.grafana.app/v0alpha1 true DummyResource
runtime example.grafana.app/v0alpha1 false RuntimeInfo
```
### Observability
Logs, metrics and traces are supported. See `--grafana.log.*`, `--grafana.metrics.*` and `--grafana.tracing.*` flags for details.
```shell
go run ./pkg/cmd/grafana apiserver \
--runtime-config=example.grafana.app/v0alpha1=true \
--help
```
For example, to enable debug logs, metrics and traces (using [self-instrumentation](../../../../devenv/docker/blocks/self-instrumentation/readme.md)) use the following:
```shell
go run ./pkg/cmd/grafana apiserver \
--runtime-config=example.grafana.app/v0alpha1=true \
--secure-port=7443 \
--grafana.log.level=debug \
--verbosity=10 \
--grafana.metrics.enable \
--grafana.tracing.jaeger.address=http://localhost:14268/api/traces \
--grafana.tracing.sampler-param=1
```

15
pkg/cmd/grafana/apiserver/cmd.go
View File

@ -7,6 +7,7 @@ import (
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/component-base/cli"
"github.com/grafana/grafana/pkg/cmd/grafana-server/commands"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/server"
"github.com/grafana/grafana/pkg/services/apiserver/standalone"
@ -64,27 +65,23 @@ func newCommandStartExampleAPIServer(o *APIServerOptions, stopCh <-chan struct{}
if err := o.RunAPIServer(config, stopCh); err != nil {
return err
}
return nil
},
}
cmd.Flags().StringVar(&runtimeConfig, "runtime-config", "", "A set of key=value pairs that enable or disable built-in APIs.")
if factoryOptions := o.factory.GetOptions(); factoryOptions != nil {
factoryOptions.AddFlags(cmd.Flags())
}
o.ExtraOptions.AddFlags(cmd.Flags())
// Register standard k8s flags with the command line
o.RecommendedOptions.AddFlags(cmd.Flags())
o.AddFlags(cmd.Flags())
return cmd
}
func RunCLI() int {
func RunCLI(opts commands.ServerOptions) int {
stopCh := genericapiserver.SetupSignalHandler()
commands.SetBuildInfo(opts)
options := newAPIServerOptions(os.Stdout, os.Stderr)
cmd := newCommandStartExampleAPIServer(options, stopCh)

125
pkg/cmd/grafana/apiserver/server.go
View File

@ -9,16 +9,17 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
utilerrors "k8s.io/apimachinery/pkg/util/errors"
genericapiserver "k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/options"
"k8s.io/client-go/tools/clientcmd"
netutils "k8s.io/utils/net"
"github.com/grafana/grafana/pkg/apiserver/builder"
"github.com/grafana/grafana/pkg/infra/log"
grafanaAPIServer "github.com/grafana/grafana/pkg/services/apiserver"
grafanaAPIServerOptions "github.com/grafana/grafana/pkg/services/apiserver/options"
"github.com/grafana/grafana/pkg/services/apiserver/standalone"
standaloneoptions "github.com/grafana/grafana/pkg/services/apiserver/standalone/options"
"github.com/grafana/grafana/pkg/services/apiserver/utils"
"github.com/grafana/grafana/pkg/setting"
"github.com/spf13/pflag"
)
const (
@ -28,25 +29,24 @@ const (
// APIServerOptions contains the state for the apiserver
type APIServerOptions struct {
factory standalone.APIServerFactory
builders []builder.APIGroupBuilder
ExtraOptions *grafanaAPIServerOptions.ExtraOptions
RecommendedOptions *options.RecommendedOptions
AlternateDNS []string
factory standalone.APIServerFactory
builders []builder.APIGroupBuilder
Options *standaloneoptions.Options
AlternateDNS []string
logger log.Logger
StdOut io.Writer
StdErr io.Writer
}
func newAPIServerOptions(out, errOut io.Writer) *APIServerOptions {
logger := log.New("grafana-apiserver")
return &APIServerOptions{
StdOut: out,
StdErr: errOut,
RecommendedOptions: options.NewRecommendedOptions(
defaultEtcdPathPrefix,
grafanaAPIServer.Codecs.LegacyCodec(), // the codec is passed to etcd and not used
),
ExtraOptions: grafanaAPIServerOptions.NewExtraOptions(),
logger: logger,
StdOut: out,
StdErr: errOut,
Options: standaloneoptions.New(logger, grafanaAPIServer.Codecs.LegacyCodec()),
}
}
@ -73,92 +73,31 @@ func (o *APIServerOptions) loadAPIGroupBuilders(apis []schema.GroupVersion) erro
return nil
}
// A copy of ApplyTo in recommended.go, but for >= 0.28, server pkg in apiserver does a bit extra causing
// a panic when CoreAPI is set to nil
func (o *APIServerOptions) ModifiedApplyTo(config *genericapiserver.RecommendedConfig) error {
if err := o.RecommendedOptions.Etcd.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.EgressSelector.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil {
return err
}
if err := o.RecommendedOptions.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil {
return err
}
if err := o.RecommendedOptions.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
return err
}
if err := o.RecommendedOptions.Audit.ApplyTo(&config.Config); err != nil {
return err
}
// TODO: determine whether we need flow control (API priority and fairness)
// We can't assume that a shared informers config was provided in standalone mode and will need a guard
// when enabling below
/* kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
if err != nil {
return err
}
if err := o.RecommendedOptions.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
return err
} */
if err := o.RecommendedOptions.CoreAPI.ApplyTo(config); err != nil {
return err
}
_, err := o.RecommendedOptions.ExtraAdmissionInitializers(config)
if err != nil {
return err
}
return nil
}
func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error) {
if err := o.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(
if err := o.Options.RecommendedOptions.SecureServing.MaybeDefaultWithSelfSignedCerts(
"localhost", o.AlternateDNS, []net.IP{netutils.ParseIPSloppy("127.0.0.1")},
); err != nil {
return nil, fmt.Errorf("error creating self-signed certificates: %v", err)
}
o.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
o.Options.RecommendedOptions.Authentication.RemoteKubeConfigFileOptional = true
// TODO: determine authorization, currently insecure because Authorization provided by recommended options doesn't work
// reason: an aggregated server won't be able to post subjectaccessreviews (Grafana doesn't have this kind)
// exact error: the server could not find the requested resource (post subjectaccessreviews.authorization.k8s.io)
o.RecommendedOptions.Authorization = nil
o.Options.RecommendedOptions.Authorization = nil
o.RecommendedOptions.Admission = nil
o.RecommendedOptions.Etcd = nil
o.Options.RecommendedOptions.Admission = nil
o.Options.RecommendedOptions.Etcd = nil
if o.RecommendedOptions.CoreAPI.CoreAPIKubeconfigPath == "" {
o.RecommendedOptions.CoreAPI = nil
if o.Options.RecommendedOptions.CoreAPI.CoreAPIKubeconfigPath == "" {
o.Options.RecommendedOptions.CoreAPI = nil
}
serverConfig := genericapiserver.NewRecommendedConfig(grafanaAPIServer.Codecs)
if o.RecommendedOptions.CoreAPI == nil {
if err := o.ModifiedApplyTo(serverConfig); err != nil {
return nil, err
}
} else {
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return nil, err
}
}
if o.ExtraOptions != nil {
if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil {
return nil, err
}
if err := o.Options.ApplyTo(serverConfig); err != nil {
return nil, fmt.Errorf("failed to apply options to server config: %w", err)
}
serverConfig.DisabledPostStartHooks = serverConfig.DisabledPostStartHooks.Insert("generic-apiserver-start-informers")
@ -177,16 +116,26 @@ func (o *APIServerOptions) Config() (*genericapiserver.RecommendedConfig, error)
return serverConfig, err
}
func (o *APIServerOptions) AddFlags(fs *pflag.FlagSet) {
o.Options.AddFlags(fs)
if factoryOptions := o.factory.GetOptions(); factoryOptions != nil {
factoryOptions.AddFlags(fs)
}
}
// Validate validates APIServerOptions
func (o *APIServerOptions) Validate() error {
errors := make([]error, 0)
// NOTE: we don't call validate on the top level recommended options as it doesn't like skipping etcd-servers
// the function is left here for troubleshooting any other config issues
// errors = append(errors, o.RecommendedOptions.Validate()...)
if factoryOptions := o.factory.GetOptions(); factoryOptions != nil {
errors = append(errors, factoryOptions.ValidateOptions()...)
}
if errs := o.Options.Validate(); len(errs) > 0 {
errors = append(errors, errors...)
}
return utilerrors.NewAggregate(errors)
}
@ -211,7 +160,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf
}
// write the local config to disk
if o.ExtraOptions.DevMode {
if o.Options.ExtraOptions.DevMode {
if err = clientcmd.WriteToFile(
utils.FormatKubeConfig(server.LoopbackClientConfig),
path.Join(dataPath, "apiserver.kubeconfig"),

9
pkg/cmd/grafana/main.go
View File

@ -41,7 +41,14 @@ func main() {
SkipFlagParsing: true,
Action: func(context *cli.Context) error {
// exit here because apiserver handles its own error output
os.Exit(apiserver.RunCLI())
os.Exit(apiserver.RunCLI(gsrv.ServerOptions{
Version: version,
Commit: commit,
EnterpriseCommit: enterpriseCommit,
BuildBranch: buildBranch,
BuildStamp: buildstamp,
Context: context,
}))
return nil
},
},

4
pkg/infra/metrics/service.go
View File

@ -59,11 +59,11 @@ func (im *InternalMetricsService) Run(ctx context.Context) error {
return ctx.Err()
}
func ProvideRegisterer(cfg *setting.Cfg) prometheus.Registerer {
func ProvideRegisterer() prometheus.Registerer {
return legacyregistry.Registerer()
}
func ProvideGatherer(cfg *setting.Cfg) prometheus.Gatherer {
func ProvideGatherer() prometheus.Gatherer {
k8sGatherer := newAddPrefixWrapper(legacyregistry.DefaultGatherer)
return newMultiRegistry(k8sGatherer, prometheus.DefaultGatherer)
}

6
pkg/infra/tracing/test_helper.go
View File

@ -16,7 +16,7 @@ func WithSpanProcessor(sp tracesdk.SpanProcessor) TracerForTestOption {
func InitializeTracerForTest(opts ...TracerForTestOption) Tracer {
exp := tracetest.NewInMemoryExporter()
tp, _ := initTracerProvider(exp, "testing", tracesdk.AlwaysSample())
tp, _ := initTracerProvider(exp, "grafana", "testing", tracesdk.AlwaysSample())
for _, opt := range opts {
opt(tp)
@ -24,7 +24,9 @@ func InitializeTracerForTest(opts ...TracerForTestOption) Tracer {
otel.SetTracerProvider(tp)
ots := &TracingService{Propagation: "jaeger,w3c", tracerProvider: tp}
cfg := NewEmptyTracingConfig()
cfg.Propagation = "jaeger,w3c"
ots := &TracingService{cfg: cfg, tracerProvider: tp}
_ = ots.initOpentelemetryTracer()
return ots
}

163
pkg/infra/tracing/tracing.go
View File

@ -6,7 +6,6 @@ import (
"math"
"net"
"net/http"
"os"
"strings"
"sync"
"time"
@ -28,7 +27,6 @@ import (
"github.com/go-kit/log/level"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/setting"
)
const (
@ -46,21 +44,11 @@ const (
)
type TracingService struct {
enabled string
Address string
Propagation string
customAttribs []attribute.KeyValue
Sampler string
SamplerParam float64
SamplerRemoteURL string
cfg *TracingConfig
log log.Logger
tracerProvider tracerProvider
trace.Tracer
Cfg *setting.Cfg
}
type tracerProvider interface {
@ -84,10 +72,9 @@ type Tracer interface {
Inject(context.Context, http.Header, trace.Span)
}
func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
ots, err := ParseSettings(cfg)
if err != nil {
return nil, err
func ProvideService(tracingCfg *TracingConfig) (*TracingService, error) {
if tracingCfg == nil {
return nil, fmt.Errorf("tracingCfg cannot be nil")
}
log.RegisterContextualLogProvider(func(ctx context.Context) ([]any, bool) {
@ -97,21 +84,18 @@ func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
return nil, false
})
ots := &TracingService{
cfg: tracingCfg,
log: log.New("tracing"),
}
if err := ots.initOpentelemetryTracer(); err != nil {
return nil, err
}
return ots, nil
}
func ParseSettings(cfg *setting.Cfg) (*TracingService, error) {
ots := &TracingService{
Cfg: cfg,
log: log.New("tracing"),
}
err := ots.parseSettings()
return ots, err
}
func (ots *TracingService) GetTracerProvider() tracerProvider {
return ots.tracerProvider
}
@ -133,96 +117,17 @@ func (noopTracerProvider) Shutdown(ctx context.Context) error {
return nil
}
func (ots *TracingService) parseSettings() error {
legacyAddress, legacyTags := "", ""
if section, err := ots.Cfg.Raw.GetSection("tracing.jaeger"); err == nil {
legacyAddress = section.Key("address").MustString("")
if legacyAddress == "" {
host, port := os.Getenv(envJaegerAgentHost), os.Getenv(envJaegerAgentPort)
if host != "" || port != "" {
legacyAddress = fmt.Sprintf("%s:%s", host, port)
}
}
legacyTags = section.Key("always_included_tag").MustString("")
ots.Sampler = section.Key("sampler_type").MustString("")
ots.SamplerParam = section.Key("sampler_param").MustFloat64(1)
ots.SamplerRemoteURL = section.Key("sampling_server_url").MustString("")
}
section := ots.Cfg.Raw.Section("tracing.opentelemetry")
var err error
// we default to legacy tag set (attributes) if the new config format is absent
ots.customAttribs, err = splitCustomAttribs(section.Key("custom_attributes").MustString(legacyTags))
if err != nil {
return err
}
// if sampler_type is set in tracing.opentelemetry, we ignore the config in tracing.jaeger
sampler := section.Key("sampler_type").MustString("")
if sampler != "" {
ots.Sampler = sampler
}
samplerParam := section.Key("sampler_param").MustFloat64(0)
if samplerParam != 0 {
ots.SamplerParam = samplerParam
}
samplerRemoteURL := section.Key("sampling_server_url").MustString("")
if samplerRemoteURL != "" {
ots.SamplerRemoteURL = samplerRemoteURL
}
section = ots.Cfg.Raw.Section("tracing.opentelemetry.jaeger")
ots.enabled = noopExporter
// we default to legacy Jaeger agent address if the new config value is empty
ots.Address = section.Key("address").MustString(legacyAddress)
ots.Propagation = section.Key("propagation").MustString("")
if ots.Address != "" {
ots.enabled = jaegerExporter
return nil
}
section = ots.Cfg.Raw.Section("tracing.opentelemetry.otlp")
ots.Address = section.Key("address").MustString("")
if ots.Address != "" {
ots.enabled = otlpExporter
}
ots.Propagation = section.Key("propagation").MustString("")
return nil
}
func (ots *TracingService) OTelExporterEnabled() bool {
return ots.enabled == otlpExporter
}
func splitCustomAttribs(s string) ([]attribute.KeyValue, error) {
res := []attribute.KeyValue{}
attribs := strings.Split(s, ",")
for _, v := range attribs {
parts := strings.SplitN(v, ":", 2)
if len(parts) > 1 {
res = append(res, attribute.String(parts[0], parts[1]))
} else if v != "" {
return nil, fmt.Errorf("custom attribute malformed - must be in 'key:value' form: %q", v)
}
}
return res, nil
}
func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
var ep jaeger.EndpointOption
// Create the Jaeger exporter: address can be either agent address (host:port) or collector URL
if strings.HasPrefix(ots.Address, "http://") || strings.HasPrefix(ots.Address, "https://") {
ots.log.Debug("using jaeger collector", "address", ots.Address)
ep = jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.Address))
} else if host, port, err := net.SplitHostPort(ots.Address); err == nil {
if strings.HasPrefix(ots.cfg.Address, "http://") || strings.HasPrefix(ots.cfg.Address, "https://") {
ots.log.Debug("using jaeger collector", "address", ots.cfg.Address)
ep = jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.cfg.Address))
} else if host, port, err := net.SplitHostPort(ots.cfg.Address); err == nil {
ots.log.Debug("using jaeger agent", "host", host, "port", port)
ep = jaeger.WithAgentEndpoint(jaeger.WithAgentHost(host), jaeger.WithAgentPort(port), jaeger.WithMaxPacketSize(64000))
} else {
return nil, fmt.Errorf("invalid tracer address: %s", ots.Address)
return nil, fmt.Errorf("invalid tracer address: %s", ots.cfg.Address)
}
exp, err := jaeger.New(ep)
if err != nil {
@ -234,10 +139,10 @@ func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider,
resource.WithAttributes(
// TODO: why are these attributes different from ones added to the
// OTLP provider?
semconv.ServiceNameKey.String("grafana"),
semconv.ServiceNameKey.String(ots.cfg.ServiceName),
attribute.String("environment", "production"),
),
resource.WithAttributes(ots.customAttribs...),
resource.WithAttributes(ots.cfg.CustomAttribs...),
)
if err != nil {
return nil, err
@ -258,7 +163,7 @@ func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider,
}
func (ots *TracingService) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.Address), otlptracegrpc.WithInsecure())
client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.cfg.Address), otlptracegrpc.WithInsecure())
exp, err := otlptrace.New(context.Background(), client)
if err != nil {
return nil, err
@ -269,39 +174,39 @@ func (ots *TracingService) initOTLPTracerProvider() (*tracesdk.TracerProvider, e
return nil, err
}
return initTracerProvider(exp, ots.Cfg.BuildVersion, sampler, ots.customAttribs...)
return initTracerProvider(exp, ots.cfg.ServiceName, ots.cfg.ServiceVersion, sampler, ots.cfg.CustomAttribs...)
}
func (ots *TracingService) initSampler() (tracesdk.Sampler, error) {
switch ots.Sampler {
switch ots.cfg.Sampler {
case "const", "":
if ots.SamplerParam >= 1 {
if ots.cfg.SamplerParam >= 1 {
return tracesdk.AlwaysSample(), nil
} else if ots.SamplerParam <= 0 {
} else if ots.cfg.SamplerParam <= 0 {
return tracesdk.NeverSample(), nil
}
return nil, fmt.Errorf("invalid param for const sampler - must be 0 or 1: %f", ots.SamplerParam)
return nil, fmt.Errorf("invalid param for const sampler - must be 0 or 1: %f", ots.cfg.SamplerParam)
case "probabilistic":
return tracesdk.TraceIDRatioBased(ots.SamplerParam), nil
return tracesdk.TraceIDRatioBased(ots.cfg.SamplerParam), nil
case "rateLimiting":
return newRateLimiter(ots.SamplerParam), nil
return newRateLimiter(ots.cfg.SamplerParam), nil
case "remote":
return jaegerremote.New("grafana",
jaegerremote.WithSamplingServerURL(ots.SamplerRemoteURL),
jaegerremote.WithInitialSampler(tracesdk.TraceIDRatioBased(ots.SamplerParam)),
jaegerremote.WithSamplingServerURL(ots.cfg.SamplerRemoteURL),
jaegerremote.WithInitialSampler(tracesdk.TraceIDRatioBased(ots.cfg.SamplerParam)),
), nil
default:
return nil, fmt.Errorf("invalid sampler type: %s", ots.Sampler)
return nil, fmt.Errorf("invalid sampler type: %s", ots.cfg.Sampler)
}
}
func initTracerProvider(exp tracesdk.SpanExporter, version string, sampler tracesdk.Sampler, customAttribs ...attribute.KeyValue) (*tracesdk.TracerProvider, error) {
func initTracerProvider(exp tracesdk.SpanExporter, serviceName string, serviceVersion string, sampler tracesdk.Sampler, customAttribs ...attribute.KeyValue) (*tracesdk.TracerProvider, error) {
res, err := resource.New(
context.Background(),
resource.WithAttributes(
semconv.ServiceNameKey.String("grafana"),
semconv.ServiceVersionKey.String(version),
semconv.ServiceNameKey.String(serviceName),
semconv.ServiceVersionKey.String(serviceVersion),
),
resource.WithAttributes(customAttribs...),
resource.WithProcessRuntimeDescription(),
@ -326,7 +231,7 @@ func (ots *TracingService) initNoopTracerProvider() (tracerProvider, error) {
func (ots *TracingService) initOpentelemetryTracer() error {
var tp tracerProvider
var err error
switch ots.enabled {
switch ots.cfg.enabled {
case jaegerExporter:
tp, err = ots.initJaegerTracerProvider()
if err != nil {
@ -347,12 +252,12 @@ func (ots *TracingService) initOpentelemetryTracer() error {
// Register our TracerProvider as the global so any imported
// instrumentation in the future will default to using it
// only if tracing is enabled
if ots.enabled != "" {
if ots.cfg.enabled != "" {
otel.SetTracerProvider(tp)
}
propagators := []propagation.TextMapPropagator{}
for _, p := range strings.Split(ots.Propagation, ",") {
for _, p := range strings.Split(ots.cfg.Propagation, ",") {
switch p {
case w3cPropagator:
propagators = append(propagators, propagation.TraceContext{}, propagation.Baggage{})

144
pkg/infra/tracing/tracing_config.go Normal file
View File

@ -0,0 +1,144 @@
package tracing
import (
"fmt"
"os"
"strings"
"github.com/grafana/grafana/pkg/setting"
"go.opentelemetry.io/otel/attribute"
)
type TracingConfig struct {
enabled string
Address string
Propagation string
CustomAttribs []attribute.KeyValue
Sampler string
SamplerParam float64
SamplerRemoteURL string
ServiceName string
ServiceVersion string
}
func ProvideTracingConfig(cfg *setting.Cfg) (*TracingConfig, error) {
return ParseTracingConfig(cfg)
}
func NewEmptyTracingConfig() *TracingConfig {
return &TracingConfig{
CustomAttribs: []attribute.KeyValue{},
}
}
func NewJaegerTracingConfig(address string, propagation string) (*TracingConfig, error) {
if address == "" {
return nil, fmt.Errorf("address cannot be empty")
}
cfg := NewEmptyTracingConfig()
cfg.enabled = jaegerExporter
cfg.Address = address
cfg.Propagation = propagation
return cfg, nil
}
func NewOTLPTracingConfig(address string, propagation string) (*TracingConfig, error) {
if address == "" {
return nil, fmt.Errorf("address cannot be empty")
}
cfg := NewEmptyTracingConfig()
cfg.enabled = otlpExporter
cfg.Address = address
cfg.Propagation = propagation
return cfg, nil
}
func ParseTracingConfig(cfg *setting.Cfg) (*TracingConfig, error) {
if cfg == nil {
return nil, fmt.Errorf("cfg cannot be nil")
}
tc := NewEmptyTracingConfig()
tc.ServiceName = "grafana"
tc.ServiceVersion = cfg.BuildVersion
legacyAddress, legacyTags := "", ""
if section, err := cfg.Raw.GetSection("tracing.jaeger"); err == nil {
legacyAddress = section.Key("address").MustString("")
if legacyAddress == "" {
host, port := os.Getenv(envJaegerAgentHost), os.Getenv(envJaegerAgentPort)
if host != "" || port != "" {
legacyAddress = fmt.Sprintf("%s:%s", host, port)
}
}
legacyTags = section.Key("always_included_tag").MustString("")
tc.Sampler = section.Key("sampler_type").MustString("")
tc.SamplerParam = section.Key("sampler_param").MustFloat64(1)
tc.SamplerRemoteURL = section.Key("sampling_server_url").MustString("")
}
section := cfg.Raw.Section("tracing.opentelemetry")
var err error
// we default to legacy tag set (attributes) if the new config format is absent
tc.CustomAttribs, err = splitCustomAttribs(section.Key("custom_attributes").MustString(legacyTags))
if err != nil {
return nil, err
}
// if sampler_type is set in tracing.opentelemetry, we ignore the config in tracing.jaeger
sampler := section.Key("sampler_type").MustString("")
if sampler != "" {
tc.Sampler = sampler
}
samplerParam := section.Key("sampler_param").MustFloat64(0)
if samplerParam != 0 {
tc.SamplerParam = samplerParam
}
samplerRemoteURL := section.Key("sampling_server_url").MustString("")
if samplerRemoteURL != "" {
tc.SamplerRemoteURL = samplerRemoteURL
}
section = cfg.Raw.Section("tracing.opentelemetry.jaeger")
tc.enabled = noopExporter
// we default to legacy Jaeger agent address if the new config value is empty
tc.Address = section.Key("address").MustString(legacyAddress)
tc.Propagation = section.Key("propagation").MustString("")
if tc.Address != "" {
tc.enabled = jaegerExporter
return tc, nil
}
section = cfg.Raw.Section("tracing.opentelemetry.otlp")
tc.Address = section.Key("address").MustString("")
if tc.Address != "" {
tc.enabled = otlpExporter
}
tc.Propagation = section.Key("propagation").MustString("")
return tc, nil
}
func (tc TracingConfig) OTelExporterEnabled() bool {
return tc.enabled == otlpExporter
}
func splitCustomAttribs(s string) ([]attribute.KeyValue, error) {
res := []attribute.KeyValue{}
attribs := strings.Split(s, ",")
for _, v := range attribs {
parts := strings.SplitN(v, ":", 2)
if len(parts) > 1 {
res = append(res, attribute.String(parts[0], parts[1]))
} else if v != "" {
return nil, fmt.Errorf("custom attribute malformed - must be in 'key:value' form: %q", v)
}
}
return res, nil
}

190
pkg/infra/tracing/tracing_config_test.go Normal file
View File

@ -0,0 +1,190 @@
package tracing
import (
"testing"
"github.com/stretchr/testify/assert"
"go.opentelemetry.io/otel/attribute"
"github.com/grafana/grafana/pkg/setting"
)
// TODO(zserge) Add proper tests for opentelemetry
func TestSplitCustomAttribs(t *testing.T) {
tests := []struct {
input string
expected []attribute.KeyValue
}{
{
input: "key1:value:1",
expected: []attribute.KeyValue{attribute.String("key1", "value:1")},
},
{
input: "key1:value1,key2:value2",
expected: []attribute.KeyValue{
attribute.String("key1", "value1"),
attribute.String("key2", "value2"),
},
},
{
input: "",
expected: []attribute.KeyValue{},
},
}
for _, test := range tests {
attribs, err := splitCustomAttribs(test.input)
assert.NoError(t, err)
assert.EqualValues(t, test.expected, attribs)
}
}
func TestSplitCustomAttribs_Malformed(t *testing.T) {
tests := []struct {
input string
}{
{input: "key1=value1"},
{input: "key1"},
}
for _, test := range tests {
_, err := splitCustomAttribs(test.input)
assert.Error(t, err)
}
}
func TestTracingConfig(t *testing.T) {
for _, test := range []struct {
Name string
Cfg string
Env map[string]string
ExpectedExporter string
ExpectedAddress string
ExpectedPropagator string
ExpectedAttrs []attribute.KeyValue
ExpectedSampler string
ExpectedSamplerParam float64
ExpectedSamplingServerURL string
}{
{
Name: "default config uses noop exporter",
Cfg: "",
ExpectedExporter: noopExporter,
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "custom attributes are parsed",
Cfg: `
[tracing.opentelemetry]
custom_attributes = key1:value1,key2:value2
`,
ExpectedExporter: noopExporter,
ExpectedAttrs: []attribute.KeyValue{attribute.String("key1", "value1"), attribute.String("key2", "value2")},
},
{
Name: "jaeger address is parsed",
Cfg: `
[tracing.opentelemetry.jaeger]
address = jaeger.example.com:6831
`,
ExpectedExporter: jaegerExporter,
ExpectedAddress: "jaeger.example.com:6831",
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "OTLP address is parsed",
Cfg: `
[tracing.opentelemetry.otlp]
address = otlp.example.com:4317
`,
ExpectedExporter: otlpExporter,
ExpectedAddress: "otlp.example.com:4317",
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "legacy config format is supported",
Cfg: `
[tracing.jaeger]
address = jaeger.example.com:6831
`,
ExpectedExporter: jaegerExporter,
ExpectedAddress: "jaeger.example.com:6831",
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "legacy env variables are supported",
Cfg: `[tracing.jaeger]`,
Env: map[string]string{
"JAEGER_AGENT_HOST": "example.com",
"JAEGER_AGENT_PORT": "12345",
},
ExpectedExporter: jaegerExporter,
ExpectedAddress: "example.com:12345",
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "opentelemetry config format is prioritised over legacy jaeger",
Cfg: `
[tracing.jaeger]
address = foo.com:6831
custom_tags = a:b
sampler_param = 0
[tracing.opentelemetry]
custom_attributes = c:d
sampler_param = 1
[tracing.opentelemetry.jaeger]
address = bar.com:6831
`,
ExpectedExporter: jaegerExporter,
ExpectedAddress: "bar.com:6831",
ExpectedAttrs: []attribute.KeyValue{attribute.String("c", "d")},
ExpectedSamplerParam: 1.0,
},
{
Name: "remote sampler config is parsed from otel config",
Cfg: `
[tracing.opentelemetry]
sampler_type = remote
sampler_param = 0.5
sampling_server_url = http://example.com:5778/sampling
[tracing.opentelemetry.otlp]
address = otlp.example.com:4317
`,
ExpectedExporter: otlpExporter,
ExpectedAddress: "otlp.example.com:4317",
ExpectedAttrs: []attribute.KeyValue{},
ExpectedSampler: "remote",
ExpectedSamplerParam: 0.5,
ExpectedSamplingServerURL: "http://example.com:5778/sampling",
},
} {
t.Run(test.Name, func(t *testing.T) {
// export environment variables
if test.Env != nil {
for k, v := range test.Env {
t.Setenv(k, v)
}
}
// parse config sections
cfg := setting.NewCfg()
err := cfg.Raw.Append([]byte(test.Cfg))
assert.NoError(t, err)
// create tracingConfig
tracingConfig, err := ProvideTracingConfig(cfg)
assert.NoError(t, err)
// make sure tracker is properly configured
assert.Equal(t, test.ExpectedExporter, tracingConfig.enabled)
assert.Equal(t, test.ExpectedAddress, tracingConfig.Address)
assert.Equal(t, test.ExpectedPropagator, tracingConfig.Propagation)
assert.Equal(t, test.ExpectedAttrs, tracingConfig.CustomAttribs)
if test.ExpectedSampler != "" {
assert.Equal(t, test.ExpectedSampler, tracingConfig.Sampler)
assert.Equal(t, test.ExpectedSamplerParam, tracingConfig.SamplerParam)
assert.Equal(t, test.ExpectedSamplingServerURL, tracingConfig.SamplerRemoteURL)
}
})
}
}

202
pkg/infra/tracing/tracing_test.go
View File

@ -5,220 +5,38 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.opentelemetry.io/otel/attribute"
"github.com/grafana/grafana/pkg/setting"
)
// TODO(zserge) Add proper tests for opentelemetry
func TestSplitCustomAttribs(t *testing.T) {
tests := []struct {
input string
expected []attribute.KeyValue
}{
{
input: "key1:value:1",
expected: []attribute.KeyValue{attribute.String("key1", "value:1")},
},
{
input: "key1:value1,key2:value2",
expected: []attribute.KeyValue{
attribute.String("key1", "value1"),
attribute.String("key2", "value2"),
},
},
{
input: "",
expected: []attribute.KeyValue{},
},
}
for _, test := range tests {
attribs, err := splitCustomAttribs(test.input)
assert.NoError(t, err)
assert.EqualValues(t, test.expected, attribs)
}
}
func TestSplitCustomAttribs_Malformed(t *testing.T) {
tests := []struct {
input string
}{
{input: "key1=value1"},
{input: "key1"},
}
for _, test := range tests {
_, err := splitCustomAttribs(test.input)
assert.Error(t, err)
}
}
func TestTracingConfig(t *testing.T) {
for _, test := range []struct {
Name string
Cfg string
Env map[string]string
ExpectedExporter string
ExpectedAddress string
ExpectedPropagator string
ExpectedAttrs []attribute.KeyValue
ExpectedSampler string
ExpectedSamplerParam float64
ExpectedSamplingServerURL string
}{
{
Name: "default config uses noop exporter",
Cfg: "",
ExpectedExporter: noopExporter,
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "custom attributes are parsed",
Cfg: `
[tracing.opentelemetry]
custom_attributes = key1:value1,key2:value2
`,
ExpectedExporter: noopExporter,
ExpectedAttrs: []attribute.KeyValue{attribute.String("key1", "value1"), attribute.String("key2", "value2")},
},
{
Name: "jaeger address is parsed",
Cfg: `
[tracing.opentelemetry.jaeger]
address = jaeger.example.com:6831
`,
ExpectedExporter: jaegerExporter,
ExpectedAddress: "jaeger.example.com:6831",
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "OTLP address is parsed",
Cfg: `
[tracing.opentelemetry.otlp]
address = otlp.example.com:4317
`,
ExpectedExporter: otlpExporter,
ExpectedAddress: "otlp.example.com:4317",
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "legacy config format is supported",
Cfg: `
[tracing.jaeger]
address = jaeger.example.com:6831
`,
ExpectedExporter: jaegerExporter,
ExpectedAddress: "jaeger.example.com:6831",
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "legacy env variables are supported",
Cfg: `[tracing.jaeger]`,
Env: map[string]string{
"JAEGER_AGENT_HOST": "example.com",
"JAEGER_AGENT_PORT": "12345",
},
ExpectedExporter: jaegerExporter,
ExpectedAddress: "example.com:12345",
ExpectedAttrs: []attribute.KeyValue{},
},
{
Name: "opentelemetry config format is prioritised over legacy jaeger",
Cfg: `
[tracing.jaeger]
address = foo.com:6831
custom_tags = a:b
sampler_param = 0
[tracing.opentelemetry]
custom_attributes = c:d
sampler_param = 1
[tracing.opentelemetry.jaeger]
address = bar.com:6831
`,
ExpectedExporter: jaegerExporter,
ExpectedAddress: "bar.com:6831",
ExpectedAttrs: []attribute.KeyValue{attribute.String("c", "d")},
ExpectedSamplerParam: 1.0,
},
{
Name: "remote sampler config is parsed from otel config",
Cfg: `
[tracing.opentelemetry]
sampler_type = remote
sampler_param = 0.5
sampling_server_url = http://example.com:5778/sampling
[tracing.opentelemetry.otlp]
address = otlp.example.com:4317
`,
ExpectedExporter: otlpExporter,
ExpectedAddress: "otlp.example.com:4317",
ExpectedAttrs: []attribute.KeyValue{},
ExpectedSampler: "remote",
ExpectedSamplerParam: 0.5,
ExpectedSamplingServerURL: "http://example.com:5778/sampling",
},
} {
t.Run(test.Name, func(t *testing.T) {
// export environment variables
if test.Env != nil {
for k, v := range test.Env {
t.Setenv(k, v)
}
}
// parse config sections
cfg := setting.NewCfg()
err := cfg.Raw.Append([]byte(test.Cfg))
assert.NoError(t, err)
// create tracer
tracer, err := ProvideService(cfg)
assert.NoError(t, err)
// make sure tracker is properly configured
assert.Equal(t, test.ExpectedExporter, tracer.enabled)
assert.Equal(t, test.ExpectedAddress, tracer.Address)
assert.Equal(t, test.ExpectedPropagator, tracer.Propagation)
assert.Equal(t, test.ExpectedAttrs, tracer.customAttribs)
if test.ExpectedSampler != "" {
assert.Equal(t, test.ExpectedSampler, tracer.Sampler)
assert.Equal(t, test.ExpectedSamplerParam, tracer.SamplerParam)
assert.Equal(t, test.ExpectedSamplingServerURL, tracer.SamplerRemoteURL)
}
})
}
}
func TestInitSampler(t *testing.T) {
otel := &TracingService{}
otel.cfg = NewEmptyTracingConfig()
sampler, err := otel.initSampler()
require.NoError(t, err)
assert.Equal(t, "AlwaysOffSampler", sampler.Description())
otel.Sampler = "bogus"
otel.cfg.Sampler = "bogus"
_, err = otel.initSampler()
require.Error(t, err)
otel.Sampler = "const"
otel.SamplerParam = 0.5
otel.cfg.Sampler = "const"
otel.cfg.SamplerParam = 0.5
_, err = otel.initSampler()
require.Error(t, err)
otel.Sampler = "const"
otel.SamplerParam = 1.0
otel.cfg.Sampler = "const"
otel.cfg.SamplerParam = 1.0
sampler, err = otel.initSampler()
require.NoError(t, err)
assert.Equal(t, "AlwaysOnSampler", sampler.Description())
otel.Sampler = "probabilistic"
otel.SamplerParam = 0.5
otel.cfg.Sampler = "probabilistic"
otel.cfg.SamplerParam = 0.5
sampler, err = otel.initSampler()
require.NoError(t, err)
assert.Equal(t, "TraceIDRatioBased{0.5}", sampler.Description())
otel.Sampler = "rateLimiting"
otel.SamplerParam = 100.25
otel.cfg.Sampler = "rateLimiting"
otel.cfg.SamplerParam = 100.25
sampler, err = otel.initSampler()
require.NoError(t, err)
assert.Equal(t, "RateLimitingSampler{100.25}", sampler.Description())

1
pkg/server/wire.go
View File

@ -247,6 +247,7 @@ var wireBasicSet = wire.NewSet(
notifications.ProvideService,
notifications.ProvideSmtpService,
tracing.ProvideService,
tracing.ProvideTracingConfig,
wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)),
testdatasource.ProvideService,
ldapapi.ProvideService,

18
pkg/services/apiserver/options/options.go
View File

@ -27,13 +27,10 @@ type Options struct {
func NewOptions(codec runtime.Codec) *Options {
return &Options{
RecommendedOptions: genericoptions.NewRecommendedOptions(
defaultEtcdPathPrefix,
codec,
),
AggregatorOptions: NewAggregatorServerOptions(),
StorageOptions: NewStorageOptions(),
ExtraOptions: NewExtraOptions(),
RecommendedOptions: NewRecommendedOptions(codec),
AggregatorOptions: NewAggregatorServerOptions(),
StorageOptions: NewStorageOptions(),
ExtraOptions: NewExtraOptions(),
}
}
@ -118,6 +115,13 @@ func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) erro
return nil
}
func NewRecommendedOptions(codec runtime.Codec) *genericoptions.RecommendedOptions {
return genericoptions.NewRecommendedOptions(
defaultEtcdPathPrefix,
codec,
)
}
type fakeListener struct {
server net.Conn
client net.Conn

41
pkg/services/apiserver/standalone/options/logging.go Normal file
View File

@ -0,0 +1,41 @@
package options
import (
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/setting"
"github.com/spf13/pflag"
genericapiserver "k8s.io/apiserver/pkg/server"
)
type LoggingOptions struct {
logger log.Logger
Level string
}
func NewLoggingOptions(logger log.Logger) *LoggingOptions {
return &LoggingOptions{
logger: logger,
}
}
func (o *LoggingOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&o.Level, "grafana.log.level", "debug", "Log level, debug, info, warn, error.")
}
func (o *LoggingOptions) Validate() []error {
return nil
}
func (o *LoggingOptions) ApplyTo(c *genericapiserver.RecommendedConfig) error {
err := log.SetupConsoleLogger(o.Level)
if err != nil {
return err
}
o.logger.Info("Starting grafana-apiserver", "version", setting.BuildVersion, "commit", setting.BuildCommit, "branch", setting.BuildBranch, "compiled", time.Unix(setting.BuildStamp, 0))
o.logger.Debug("Console logging initialized", "logLevel", o.Level)
return nil
}

42
pkg/services/apiserver/standalone/options/metrics.go Normal file
View File

@ -0,0 +1,42 @@
package options
import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus"
"github.com/spf13/pflag"
genericapiserver "k8s.io/apiserver/pkg/server"
)
type MetricsOptions struct {
logger log.Logger
Enabled bool
MetricsRegisterer prometheus.Registerer
}
func NewMetrcicsOptions(logger log.Logger) *MetricsOptions {
return &MetricsOptions{
logger: logger,
}
}
func (o *MetricsOptions) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&o.Enabled, "grafana.metrics.enable", false, "Enable metrics and Prometheus /metrics endpoint.")
}
func (o *MetricsOptions) Validate() []error {
return nil
}
func (o *MetricsOptions) ApplyTo(c *genericapiserver.RecommendedConfig) error {
c.EnableMetrics = o.Enabled
o.MetricsRegisterer = metrics.ProvideRegisterer()
metrics.SetBuildInformation(o.MetricsRegisterer, setting.BuildVersion, setting.BuildCommit, setting.BuildBranch, setting.BuildStamp)
if o.Enabled {
o.logger.Debug("Metrics enabled")
}
return nil
}

159
pkg/services/apiserver/standalone/options/options.go Normal file
View File

@ -0,0 +1,159 @@
package options
import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/apiserver/options"
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/runtime"
genericapiserver "k8s.io/apiserver/pkg/server"
genericoptions "k8s.io/apiserver/pkg/server/options"
)
type Options struct {
LoggingOptions *LoggingOptions
ExtraOptions *options.ExtraOptions
RecommendedOptions *genericoptions.RecommendedOptions
TracingOptions *TracingOptions
MetricsOptions *MetricsOptions
}
func New(logger log.Logger, codec runtime.Codec) *Options {
return &Options{
LoggingOptions: NewLoggingOptions(logger),
ExtraOptions: options.NewExtraOptions(),
RecommendedOptions: options.NewRecommendedOptions(codec),
TracingOptions: NewTracingOptions(logger),
MetricsOptions: NewMetrcicsOptions(logger),
}
}
func (o *Options) AddFlags(fs *pflag.FlagSet) {
o.LoggingOptions.AddFlags(fs)
o.ExtraOptions.AddFlags(fs)
o.RecommendedOptions.AddFlags(fs)
o.TracingOptions.AddFlags(fs)
o.MetricsOptions.AddFlags(fs)
}
func (o *Options) Validate() []error {
if errs := o.LoggingOptions.Validate(); len(errs) != 0 {
return errs
}
if errs := o.ExtraOptions.Validate(); len(errs) != 0 {
return errs
}
if errs := o.TracingOptions.Validate(); len(errs) != 0 {
return errs
}
if errs := o.MetricsOptions.Validate(); len(errs) != 0 {
return errs
}
// NOTE: we don't call validate on the top level recommended options as it doesn't like skipping etcd-servers
// the function is left here for troubleshooting any other config issues
// errors = append(errors, o.RecommendedOptions.Validate()...)
if errs := o.RecommendedOptions.SecureServing.Validate(); len(errs) != 0 {
return errs
}
if o.ExtraOptions.DevMode {
// NOTE: Only consider authn for dev mode - resolves the failure due to missing extension apiserver auth-config
// in parent k8s
if errs := o.RecommendedOptions.Authentication.Validate(); len(errs) != 0 {
return errs
}
}
return nil
}
// A copy of ApplyTo in recommended.go, but for >= 0.28, server pkg in apiserver does a bit extra causing
// a panic when CoreAPI is set to nil
func (o *Options) ModifiedApplyTo(config *genericapiserver.RecommendedConfig) error {
if err := o.RecommendedOptions.Etcd.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.EgressSelector.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.Traces.ApplyTo(config.Config.EgressSelector, &config.Config); err != nil {
return err
}
if err := o.RecommendedOptions.SecureServing.ApplyTo(&config.Config.SecureServing, &config.Config.LoopbackClientConfig); err != nil {
return err
}
if err := o.RecommendedOptions.Authentication.ApplyTo(&config.Config.Authentication, config.SecureServing, config.OpenAPIConfig); err != nil {
return err
}
if err := o.RecommendedOptions.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
return err
}
if err := o.RecommendedOptions.Audit.ApplyTo(&config.Config); err != nil {
return err
}
// TODO: determine whether we need flow control (API priority and fairness)
// We can't assume that a shared informers config was provided in standalone mode and will need a guard
// when enabling below
/* kubeClient, err := kubernetes.NewForConfig(config.ClientConfig)
if err != nil {
return err
}
if err := o.RecommendedOptions.Features.ApplyTo(&config.Config, kubeClient, config.SharedInformerFactory); err != nil {
return err
} */
if err := o.RecommendedOptions.CoreAPI.ApplyTo(config); err != nil {
return err
}
_, err := o.RecommendedOptions.ExtraAdmissionInitializers(config)
if err != nil {
return err
}
return nil
}
func (o *Options) ApplyTo(serverConfig *genericapiserver.RecommendedConfig) error {
if o.LoggingOptions != nil {
if err := o.LoggingOptions.ApplyTo(serverConfig); err != nil {
return err
}
}
if o.ExtraOptions != nil {
if err := o.ExtraOptions.ApplyTo(serverConfig); err != nil {
return err
}
}
if o.RecommendedOptions.CoreAPI == nil {
if err := o.ModifiedApplyTo(serverConfig); err != nil {
return err
}
} else {
if err := o.RecommendedOptions.ApplyTo(serverConfig); err != nil {
return err
}
}
if o.TracingOptions != nil {
if err := o.TracingOptions.ApplyTo(serverConfig); err != nil {
return err
}
}
if o.MetricsOptions != nil {
if err := o.MetricsOptions.ApplyTo(serverConfig); err != nil {
return err
}
}
return nil
}

124
pkg/services/apiserver/standalone/options/tracing.go Normal file
View File

@ -0,0 +1,124 @@
package options
import (
"context"
"errors"
"fmt"
"net/url"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/setting"
"github.com/spf13/pflag"
"go.opentelemetry.io/otel/attribute"
genericapiserver "k8s.io/apiserver/pkg/server"
)
type TracingOptions struct {
logger log.Logger
JaegerAddress string
JaegerPropagation string
OTLPAddress string
OTLPPropagation string
Tags map[string]string
SamplerType string
SamplerParam float64
SamplingServiceURL string
TracingService *tracing.TracingService
}
func NewTracingOptions(logger log.Logger) *TracingOptions {
return &TracingOptions{
logger: logger,
Tags: map[string]string{},
}
}
func (o *TracingOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&o.JaegerAddress, "grafana.tracing.jaeger.address", "", "Tracing Jaeger exporter destination, e.g. http://localhost:14268/api/traces. This enabled the Jaeger export and takes presedence over grafana.tracing.otlp.")
fs.StringVar(&o.JaegerPropagation, "grafana.tracing.jaeger.propagation", "jaeger", "Tracing Jaeger propagation specifies the text map propagation format, w3c or jaeger.")
fs.StringVar(&o.OTLPAddress, "grafana.tracing.otlp.address", "", "Tracing OTLP exporter destination, e.g. localhost:4317.")
fs.StringVar(&o.OTLPPropagation, "grafana.tracing.otlp.propagation", "w3c", "Tracing OTLP propagation specifies the text map propagation format, w3c or jaeger.")
fs.StringToStringVar(&o.Tags, "grafana.tracing.tag", map[string]string{}, "Tracing server tag in 'key=value' format. Specify multiple times to add many.")
fs.StringVar(&o.SamplerType, "grafana.tracing.sampler-type", "const", "Tracing sampler type specifies the type of the sampler: const, probabilistic, rateLimiting, or remote.")
fs.Float64Var(&o.SamplerParam, "grafana.tracing.sampler-param", 0, "Tracing sampler configuration parameter. For 'const' sampler, 0 or 1 for always false/true respectively. For 'rateLimiting' sampler, the number of spans per second. For 'remote' sampler, param is the same as for 'probabilistic' and indicates the initial sampling rate before the actual one is received from the sampling service.")
fs.StringVar(&o.SamplingServiceURL, "grafana.tracing.sampling-service", "", "Tracing server sampling service URL (used for both Jaeger and OTLP) if grafana.tracing.sampler-type=remote.")
}
func (o *TracingOptions) Validate() []error {
errors := []error{}
if o.JaegerAddress != "" {
if _, err := url.Parse(o.JaegerAddress); err != nil {
errors = append(errors, fmt.Errorf("failed to parse tracing.jaeger.address: %w", err))
}
}
if o.SamplingServiceURL != "" {
if _, err := url.Parse(o.SamplingServiceURL); err != nil {
errors = append(errors, fmt.Errorf("failed to parse tracing.sampling-service: %w", err))
}
}
return errors
}
func (o *TracingOptions) ApplyTo(config *genericapiserver.RecommendedConfig) error {
tracingCfg := tracing.NewEmptyTracingConfig()
var err error
if o.OTLPAddress != "" {
tracingCfg, err = tracing.NewOTLPTracingConfig(o.OTLPAddress, o.OTLPPropagation)
}
if o.JaegerAddress != "" {
tracingCfg, err = tracing.NewJaegerTracingConfig(o.JaegerAddress, o.JaegerPropagation)
}
if err != nil {
return err
}
tracingCfg.ServiceName = "grafana-apiserver"
tracingCfg.ServiceVersion = setting.BuildVersion
for k, v := range o.Tags {
tracingCfg.CustomAttribs = append(tracingCfg.CustomAttribs, attribute.String(k, v))
}
tracingCfg.Sampler = o.SamplerType
tracingCfg.SamplerParam = o.SamplerParam
tracingCfg.SamplerRemoteURL = o.SamplingServiceURL
ts, err := tracing.ProvideService(tracingCfg)
if err != nil {
return err
}
o.TracingService = ts
config.TracerProvider = ts.GetTracerProvider()
config.AddPostStartHookOrDie("grafana-tracing-service", func(hookCtx genericapiserver.PostStartHookContext) error {
ctx, cancel := context.WithCancel(context.Background())
go func() {
<-hookCtx.StopCh
cancel()
}()
go func() {
if err := ts.Run(ctx); err != nil && !errors.Is(err, context.Canceled) {
o.logger.Error("failed to shutdown tracing service", "error", err)
}
}()
return nil
})
return nil
}

14
pkg/services/pluginsintegration/pluginconfig/tracing.go
View File

@ -11,20 +11,20 @@ import (
// newTracingCfg creates a plugins tracing configuration based on the provided Grafana tracing config.
// If OpenTelemetry (OTLP) is disabled, a zero-value OpenTelemetryCfg is returned.
func newTracingCfg(grafanaCfg *setting.Cfg) (pCfg.Tracing, error) {
ots, err := tracing.ParseSettings(grafanaCfg)
tracingCfg, err := tracing.ParseTracingConfig(grafanaCfg)
if err != nil {
return pCfg.Tracing{}, fmt.Errorf("parse settings: %w", err)
}
if !ots.OTelExporterEnabled() {
if !tracingCfg.OTelExporterEnabled() {
return pCfg.Tracing{}, nil
}
return pCfg.Tracing{
OpenTelemetry: pCfg.OpenTelemetryCfg{
Address: ots.Address,
Propagation: ots.Propagation,
Sampler: ots.Sampler,
SamplerParam: ots.SamplerParam,
SamplerRemoteURL: ots.SamplerRemoteURL,
Address: tracingCfg.Address,
Propagation: tracingCfg.Propagation,
Sampler: tracingCfg.Sampler,
SamplerParam: tracingCfg.SamplerParam,
SamplerRemoteURL: tracingCfg.SamplerRemoteURL,
},
}, nil
}

7
pkg/services/store/entity/server/service.go
View File

@ -57,7 +57,12 @@ func ProvideService(
cfg *setting.Cfg,
features featuremgmt.FeatureToggles,
) (*service, error) {
tracing, err := tracing.ProvideService(cfg)
tracingCfg, err := tracing.ProvideTracingConfig(cfg)
if err != nil {
return nil, err
}
tracing, err := tracing.ProvideService(tracingCfg)
if err != nil {
return nil, err
}