IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

K8s: Refactor metrics to share k8s registry (#79106)

Browse Source
This commit is contained in:
Todd Treece 2023-12-06 16:34:23 -05:00 committed by GitHub
parent c631261681
commit 2a2a132c61
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 197 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
pkg/api/http_server.go
View File

@ -213,6 +213,7 @@ type HTTPServer struct {
authnService authn.Service
starApi *starApi.API
promRegister prometheus.Registerer
promGatherer prometheus.Gatherer
clientConfigProvider grafanaapiserver.DirectRestConfigProvider
namespacer request.NamespaceMapper
anonService anonymous.Service
@ -256,7 +257,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
loginAttemptService loginAttempt.Service, orgService org.Service, teamService team.Service,
accesscontrolService accesscontrol.Service, navTreeService navtree.Service,
annotationRepo annotations.Repository, tagService tag.Service, searchv2HTTPService searchV2.SearchHTTPService, oauthTokenService oauthtoken.OAuthTokenService,
statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service,
statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service, promGatherer prometheus.Gatherer,
starApi *starApi.API, promRegister prometheus.Registerer, clientConfigProvider grafanaapiserver.DirectRestConfigProvider, anonService anonymous.Service,
) (*HTTPServer, error) {
web.Env = cfg.Env
@ -356,6 +357,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
pluginsCDNService: pluginsCDNService,
starApi: starApi,
promRegister: promRegister,
promGatherer: promGatherer,
clientConfigProvider: clientConfigProvider,
namespacer: request.GetNamespaceMapper(cfg),
anonService: anonService,
@ -720,7 +722,7 @@ func (hs *HTTPServer) metricsEndpoint(ctx *web.Context) {
}
promhttp.
HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{EnableOpenMetrics: true}).
HandlerFor(hs.promGatherer, promhttp.HandlerOpts{EnableOpenMetrics: true}).
ServeHTTP(ctx.Resp, ctx.Req)
}

11
pkg/cmd/grafana-server/commands/buildinfo.go
View File

@ -5,22 +5,23 @@ import (
"time"
"github.com/grafana/grafana/pkg/extensions"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/setting"
)
func setBuildInfo(opts ServerOptions) {
func getBuildstamp(opts ServerOptions) int64 {
buildstampInt64, err := strconv.ParseInt(opts.BuildStamp, 10, 64)
if err != nil || buildstampInt64 == 0 {
buildstampInt64 = time.Now().Unix()
}
return buildstampInt64
}
func setBuildInfo(opts ServerOptions) {
setting.BuildVersion = opts.Version
setting.BuildCommit = opts.Commit
setting.EnterpriseBuildCommit = opts.EnterpriseCommit
setting.BuildStamp = buildstampInt64
setting.BuildStamp = getBuildstamp(opts)
setting.BuildBranch = opts.BuildBranch
setting.IsEnterprise = extensions.IsEnterprise
setting.Packaging = validPackaging(Packaging)
metrics.SetBuildInformation(opts.Version, opts.Commit, opts.BuildBranch, buildstampInt64)
}

3
pkg/cmd/grafana-server/commands/cli.go
View File

@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana/pkg/api"
gcli "github.com/grafana/grafana/pkg/cmd/grafana-cli/commands"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/infra/process"
"github.com/grafana/grafana/pkg/server"
_ "github.com/grafana/grafana/pkg/services/alerting/conditions"
@ -111,6 +112,8 @@ func RunServer(opts ServerOptions) error {
return err
}
metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
s, err := server.Initialize(
cfg,
server.Options{

3
pkg/cmd/grafana-server/commands/target.go
View File

@ -11,6 +11,7 @@ import (
"github.com/grafana/grafana/pkg/api"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/server"
"github.com/grafana/grafana/pkg/setting"
)
@ -88,6 +89,8 @@ func RunTargetServer(opts ServerOptions) error {
return err
}
metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
s, err := server.InitializeModuleServer(
cfg,
server.Options{

18
pkg/infra/metrics/frontendmetrics.go
View File

@ -19,7 +19,7 @@ type FrontendMetricsRecorder func(event FrontendMetricEvent)
// FrontendMetrics contains all the valid frontend metrics and a handler function for recording events
var FrontendMetrics map[string]FrontendMetricsRecorder = map[string]FrontendMetricsRecorder{}
func registerFrontendHistogram(name string, help string) {
func registerFrontendHistogram(reg prometheus.Registerer, name string, help string) {
defBuckets := []float64{.1, .25, .5, 1, 1.5, 2, 5, 10, 20, 40}
histogram := prometheus.NewHistogram(prometheus.HistogramOpts{
@ -33,14 +33,14 @@ func registerFrontendHistogram(name string, help string) {
histogram.Observe(event.Value)
}
prometheus.MustRegister(histogram)
reg.MustRegister(histogram)
}
func initFrontendMetrics() {
registerFrontendHistogram("frontend_boot_load_time_seconds", "Frontend boot time measurement")
registerFrontendHistogram("frontend_boot_first_paint_time_seconds", "Frontend boot first paint")
registerFrontendHistogram("frontend_boot_first_contentful_paint_time_seconds", "Frontend boot first contentful paint")
registerFrontendHistogram("frontend_boot_js_done_time_seconds", "Frontend boot initial js load")
registerFrontendHistogram("frontend_boot_css_time_seconds", "Frontend boot initial css load")
registerFrontendHistogram("frontend_plugins_preload_ms", "Frontend preload plugin time measurement")
func initFrontendMetrics(r prometheus.Registerer) {
registerFrontendHistogram(r, "frontend_boot_load_time_seconds", "Frontend boot time measurement")
registerFrontendHistogram(r, "frontend_boot_first_paint_time_seconds", "Frontend boot first paint")
registerFrontendHistogram(r, "frontend_boot_first_contentful_paint_time_seconds", "Frontend boot first contentful paint")
registerFrontendHistogram(r, "frontend_boot_js_done_time_seconds", "Frontend boot initial js load")
registerFrontendHistogram(r, "frontend_boot_css_time_seconds", "Frontend boot initial css load")
registerFrontendHistogram(r, "frontend_plugins_preload_ms", "Frontend preload plugin time measurement")
}

12
pkg/infra/metrics/metrics.go
View File

@ -613,7 +613,7 @@ func init() {
}
// SetBuildInformation sets the build information for this binary
func SetBuildInformation(version, revision, branch string, buildTimestamp int64) {
func SetBuildInformation(reg prometheus.Registerer, version, revision, branch string, buildTimestamp int64) {
edition := "oss"
if setting.IsEnterprise {
edition = "enterprise"
@ -631,7 +631,7 @@ func SetBuildInformation(version, revision, branch string, buildTimestamp int64)
Namespace: ExporterName,
}, []string{"version", "revision", "branch", "goversion", "edition"})
prometheus.MustRegister(grafanaBuildVersion, grafanaBuildTimestamp)
reg.MustRegister(grafanaBuildVersion, grafanaBuildTimestamp)
grafanaBuildVersion.WithLabelValues(version, revision, branch, runtime.Version(), edition).Set(1)
grafanaBuildTimestamp.WithLabelValues(version, revision, branch, runtime.Version(), edition).Set(float64(buildTimestamp))
@ -639,7 +639,7 @@ func SetBuildInformation(version, revision, branch string, buildTimestamp int64)
// SetEnvironmentInformation exposes environment values provided by the operators as an `_info` metric.
// If there are no environment metrics labels configured, this metric will not be exposed.
func SetEnvironmentInformation(labels map[string]string) error {
func SetEnvironmentInformation(reg prometheus.Registerer, labels map[string]string) error {
if len(labels) == 0 {
return nil
}
@ -651,7 +651,7 @@ func SetEnvironmentInformation(labels map[string]string) error {
ConstLabels: labels,
})
prometheus.MustRegister(grafanaEnvironmentInfo)
reg.MustRegister(grafanaEnvironmentInfo)
grafanaEnvironmentInfo.Set(1)
return nil
@ -661,8 +661,8 @@ func SetPluginBuildInformation(pluginID, pluginType, version, signatureStatus st
grafanaPluginBuildInfoDesc.WithLabelValues(pluginID, pluginType, version, signatureStatus).Set(1)
}
func initMetricVars() {
prometheus.MustRegister(
func initMetricVars(reg prometheus.Registerer) {
reg.MustRegister(
MInstanceStart,
MPageStatus,
MApiStatus,

81
pkg/infra/metrics/service.go
View File

@ -2,11 +2,17 @@ package metrics
import (
"context"
"errors"
"regexp"
"github.com/prometheus/client_golang/prometheus"
dto "github.com/prometheus/client_model/go"
"k8s.io/component-base/metrics/legacyregistry"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics/graphitebridge"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus"
)
var metricsLogger log.Logger = log.New("metrics")
@ -19,12 +25,10 @@ func (lw *logWrapper) Println(v ...any) {
lw.logger.Info("graphite metric bridge", v...)
}
func init() {
initMetricVars()
initFrontendMetrics()
}
func ProvideService(cfg *setting.Cfg, reg prometheus.Registerer) (*InternalMetricsService, error) {
initMetricVars(reg)
initFrontendMetrics(reg)
func ProvideService(cfg *setting.Cfg) (*InternalMetricsService, error) {
s := &InternalMetricsService{
Cfg: cfg,
}
@ -55,5 +59,66 @@ func (im *InternalMetricsService) Run(ctx context.Context) error {
return ctx.Err()
}
func ProvideRegisterer() prometheus.Registerer { return prometheus.DefaultRegisterer }
func ProvideRegistererForTest() prometheus.Registerer { return prometheus.NewRegistry() }
func ProvideRegisterer(cfg *setting.Cfg) prometheus.Registerer {
if cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer) {
return legacyregistry.Registerer()
}
return prometheus.DefaultRegisterer
}
func ProvideGatherer(cfg *setting.Cfg) prometheus.Gatherer {
if cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer) {
return newAddPrefixWrapper(legacyregistry.DefaultGatherer)
}
return prometheus.DefaultGatherer
}
func ProvideRegistererForTest() prometheus.Registerer {
return prometheus.NewRegistry()
}
func ProvideGathererForTest(reg prometheus.Registerer) prometheus.Gatherer {
// the registerer provided by ProvideRegistererForTest
// is a *prometheus.Registry, so it also implements prometheus.Gatherer
return reg.(*prometheus.Registry)
}
var _ prometheus.Gatherer = (*addPrefixWrapper)(nil)
// addPrefixWrapper wraps a prometheus.Gatherer, and ensures that all metric names are prefixed with `grafana_`.
// metrics with the prefix `grafana_` or `go_` are not modified.
type addPrefixWrapper struct {
orig prometheus.Gatherer
reg *regexp.Regexp
}
func newAddPrefixWrapper(orig prometheus.Gatherer) *addPrefixWrapper {
return &addPrefixWrapper{
orig: orig,
reg: regexp.MustCompile("^((?:grafana_|go_).*)"),
}
}
func (g *addPrefixWrapper) Gather() ([]*dto.MetricFamily, error) {
mf, err := g.orig.Gather()
if err != nil {
return nil, err
}
names := make(map[string]struct{})
for i := 0; i < len(mf); i++ {
m := mf[i]
if m.Name != nil && !g.reg.MatchString(*m.Name) {
*m.Name = "grafana_" + *m.Name
// since we are modifying the name, we need to check for duplicates in the gatherer
if _, exists := names[*m.Name]; exists {
return nil, errors.New("duplicate metric name: " + *m.Name)
}
}
// keep track of names to detect duplicates
names[*m.Name] = struct{}{}
}
return mf, nil
}

61
pkg/infra/metrics/service_test.go Normal file
View File

@ -0,0 +1,61 @@
package metrics
import (
"testing"
dto "github.com/prometheus/client_model/go"
"github.com/stretchr/testify/require"
)
func TestK8sGathererWrapper_Gather(t *testing.T) {
orig := &mockGatherer{}
g := newAddPrefixWrapper(orig)
t.Run("metrics with grafana and go prefix are not modified", func(t *testing.T) {
originalMF := []*dto.MetricFamily{
{Name: strptr("grafana_metric1")},
{Name: strptr("metric2")},
{Name: strptr("go_metric1")},
}
orig.GatherFunc = func() ([]*dto.MetricFamily, error) {
return originalMF, nil
}
expectedMF := []*dto.MetricFamily{
{Name: strptr("grafana_metric1")},
{Name: strptr("grafana_metric2")},
{Name: strptr("go_metric1")},
}
mf, err := g.Gather()
require.NoError(t, err)
require.Equal(t, expectedMF, mf)
})
t.Run("duplicate metrics result in an error", func(t *testing.T) {
originalMF := []*dto.MetricFamily{
{Name: strptr("grafana_metric1")},
{Name: strptr("metric1")},
}
orig.GatherFunc = func() ([]*dto.MetricFamily, error) {
return originalMF, nil
}
_, err := g.Gather()
require.Error(t, err)
})
}
type mockGatherer struct {
GatherFunc func() ([]*dto.MetricFamily, error)
}
func (m *mockGatherer) Gather() ([]*dto.MetricFamily, error) {
return m.GatherFunc()
}
func strptr(s string) *string {
return &s
}

17
pkg/infra/metrics/wireset.go Normal file
View File

@ -0,0 +1,17 @@
package metrics
import (
"github.com/google/wire"
)
var WireSet = wire.NewSet(
ProvideService,
ProvideRegisterer,
ProvideGatherer,
)
var WireSetForTest = wire.NewSet(
ProvideService,
ProvideRegistererForTest,
ProvideGathererForTest,
)

10
pkg/server/server.go
View File

@ -13,6 +13,8 @@ import (
"golang.org/x/sync/errgroup"
"github.com/prometheus/client_golang/prometheus"
"github.com/grafana/grafana/pkg/api"
_ "github.com/grafana/grafana/pkg/extensions"
"github.com/grafana/grafana/pkg/infra/log"
@ -38,9 +40,10 @@ type Options struct {
func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry,
provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry,
usageStatsProvidersRegistry registry.UsageStatsProvidersRegistry, statsCollectorService *statscollector.Service,
promReg prometheus.Registerer,
) (*Server, error) {
statsCollectorService.RegisterProviders(usageStatsProvidersRegistry.GetServices())
s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider)
s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider, promReg)
if err != nil {
return nil, err
}
@ -54,11 +57,13 @@ func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistr
func newServer(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry,
provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry,
promReg prometheus.Registerer,
) (*Server, error) {
rootCtx, shutdownFn := context.WithCancel(context.Background())
childRoutines, childCtx := errgroup.WithContext(rootCtx)
s := &Server{
promReg: promReg,
context: childCtx,
childRoutines: childRoutines,
HTTPServer: httpServer,
@ -101,6 +106,7 @@ type Server struct {
HTTPServer *api.HTTPServer
roleRegistry accesscontrol.RoleRegistry
provisioningService provisioning.ProvisioningService
promReg prometheus.Registerer
}
// Init initializes the server and its services.
@ -117,7 +123,7 @@ func (s *Server) Init() error {
return err
}
if err := metrics.SetEnvironmentInformation(s.cfg.MetricsGrafanaEnvironmentInfo); err != nil {
if err := metrics.SetEnvironmentInformation(s.promReg, s.cfg.MetricsGrafanaEnvironmentInfo); err != nil {
return err
}

3
pkg/server/server_test.go
View File

@ -7,6 +7,7 @@ import (
"testing"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/registry"
@ -48,7 +49,7 @@ func (s *testService) IsDisabled() bool {
func testServer(t *testing.T, services ...registry.BackgroundService) *Server {
t.Helper()
s, err := newServer(Options{}, setting.NewCfg(), nil, &acimpl.Service{}, nil, backgroundsvcs.NewBackgroundServiceRegistry(services...))
s, err := newServer(Options{}, setting.NewCfg(), nil, &acimpl.Service{}, nil, backgroundsvcs.NewBackgroundServiceRegistry(services...), prometheus.NewRegistry())
require.NoError(t, err)
// Required to skip configuration initialization that causes
// DI errors in this test.

6
pkg/server/wire.go
View File

@ -254,7 +254,6 @@ var wireBasicSet = wire.NewSet(
notifications.ProvideSmtpService,
tracing.ProvideService,
wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)),
metrics.ProvideService,
testdatasource.ProvideService,
ldapapi.ProvideService,
opentsdb.ProvideService,
@ -389,7 +388,7 @@ var wireBasicSet = wire.NewSet(
var wireSet = wire.NewSet(
wireBasicSet,
metrics.ProvideRegisterer,
metrics.WireSet,
sqlstore.ProvideService,
ngmetrics.ProvideService,
wire.Bind(new(notifications.Service), new(*notifications.NotificationService)),
@ -404,6 +403,7 @@ var wireSet = wire.NewSet(
var wireCLISet = wire.NewSet(
NewRunner,
wireBasicSet,
metrics.WireSet,
sqlstore.ProvideService,
ngmetrics.ProvideService,
wire.Bind(new(notifications.Service), new(*notifications.NotificationService)),
@ -418,7 +418,7 @@ var wireCLISet = wire.NewSet(
var wireTestSet = wire.NewSet(
wireBasicSet,
ProvideTestEnv,
metrics.ProvideRegistererForTest,
metrics.WireSetForTest,
sqlstore.ProvideServiceForTests,
ngmetrics.ProvideServiceForTest,
notifications.MockNotificationService,

2
pkg/server/wireexts_oss.go
View File

@ -7,6 +7,7 @@ package server
import (
"github.com/google/wire"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/plugins/manager"
"github.com/grafana/grafana/pkg/registry"
@ -123,6 +124,7 @@ var wireExtsTestSet = wire.NewSet(
var wireExtsBaseCLISet = wire.NewSet(
NewModuleRunner,
metrics.WireSet,
featuremgmt.ProvideManagerService,
featuremgmt.ProvideToggles,
hooks.ProvideService,