K8s: Refactor metrics to share k8s registry (#79106)

pkg/api/http_server.go

@@ -213,6 +213,7 @@ type HTTPServer struct {
 	authnService         authn.Service
 	starApi              *starApi.API
 	promRegister         prometheus.Registerer
+	promGatherer         prometheus.Gatherer
 	clientConfigProvider grafanaapiserver.DirectRestConfigProvider
 	namespacer           request.NamespaceMapper
 	anonService          anonymous.Service
@@ -256,7 +257,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 	loginAttemptService loginAttempt.Service, orgService org.Service, teamService team.Service,
 	accesscontrolService accesscontrol.Service, navTreeService navtree.Service,
 	annotationRepo annotations.Repository, tagService tag.Service, searchv2HTTPService searchV2.SearchHTTPService, oauthTokenService oauthtoken.OAuthTokenService,
-	statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service,
+	statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service, promGatherer prometheus.Gatherer,
 	starApi *starApi.API, promRegister prometheus.Registerer, clientConfigProvider grafanaapiserver.DirectRestConfigProvider, anonService anonymous.Service,
 ) (*HTTPServer, error) {
 	web.Env = cfg.Env
@@ -356,6 +357,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
 		pluginsCDNService:            pluginsCDNService,
 		starApi:                      starApi,
 		promRegister:                 promRegister,
+		promGatherer:                 promGatherer,
 		clientConfigProvider:         clientConfigProvider,
 		namespacer:                   request.GetNamespaceMapper(cfg),
 		anonService:                  anonService,
@@ -720,7 +722,7 @@ func (hs *HTTPServer) metricsEndpoint(ctx *web.Context) {
 	}
 
 	promhttp.
-		HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{EnableOpenMetrics: true}).
+		HandlerFor(hs.promGatherer, promhttp.HandlerOpts{EnableOpenMetrics: true}).
 		ServeHTTP(ctx.Resp, ctx.Req)
 }
 

pkg/cmd/grafana-server/commands/buildinfo.go

@@ -5,22 +5,23 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/extensions"
-	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/setting"
 )
 
-func setBuildInfo(opts ServerOptions) {
+func getBuildstamp(opts ServerOptions) int64 {
 	buildstampInt64, err := strconv.ParseInt(opts.BuildStamp, 10, 64)
 	if err != nil || buildstampInt64 == 0 {
 		buildstampInt64 = time.Now().Unix()
 	}
+	return buildstampInt64
+}
+
+func setBuildInfo(opts ServerOptions) {
 	setting.BuildVersion = opts.Version
 	setting.BuildCommit = opts.Commit
 	setting.EnterpriseBuildCommit = opts.EnterpriseCommit
-	setting.BuildStamp = buildstampInt64
+	setting.BuildStamp = getBuildstamp(opts)
 	setting.BuildBranch = opts.BuildBranch
 	setting.IsEnterprise = extensions.IsEnterprise
 	setting.Packaging = validPackaging(Packaging)
-
-	metrics.SetBuildInformation(opts.Version, opts.Commit, opts.BuildBranch, buildstampInt64)
 }

pkg/cmd/grafana-server/commands/cli.go

@@ -16,6 +16,7 @@ import (
 	"github.com/grafana/grafana/pkg/api"
 	gcli "github.com/grafana/grafana/pkg/cmd/grafana-cli/commands"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/infra/process"
 	"github.com/grafana/grafana/pkg/server"
 	_ "github.com/grafana/grafana/pkg/services/alerting/conditions"
@@ -111,6 +112,8 @@ func RunServer(opts ServerOptions) error {
 		return err
 	}
 
+	metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
+
 	s, err := server.Initialize(
 		cfg,
 		server.Options{

pkg/cmd/grafana-server/commands/target.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/api"
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/server"
 	"github.com/grafana/grafana/pkg/setting"
 )
@@ -88,6 +89,8 @@ func RunTargetServer(opts ServerOptions) error {
 		return err
 	}
 
+	metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts))
+
 	s, err := server.InitializeModuleServer(
 		cfg,
 		server.Options{

pkg/infra/metrics/frontendmetrics.go

@@ -19,7 +19,7 @@ type FrontendMetricsRecorder func(event FrontendMetricEvent)
 // FrontendMetrics contains all the valid frontend metrics and a handler function for recording events
 var FrontendMetrics map[string]FrontendMetricsRecorder = map[string]FrontendMetricsRecorder{}
 
-func registerFrontendHistogram(name string, help string) {
+func registerFrontendHistogram(reg prometheus.Registerer, name string, help string) {
 	defBuckets := []float64{.1, .25, .5, 1, 1.5, 2, 5, 10, 20, 40}
 
 	histogram := prometheus.NewHistogram(prometheus.HistogramOpts{
@@ -33,14 +33,14 @@ func registerFrontendHistogram(name string, help string) {
 		histogram.Observe(event.Value)
 	}
 
-	prometheus.MustRegister(histogram)
+	reg.MustRegister(histogram)
 }
 
-func initFrontendMetrics() {
+func initFrontendMetrics(r prometheus.Registerer) {
-	registerFrontendHistogram("frontend_boot_load_time_seconds", "Frontend boot time measurement")
+	registerFrontendHistogram(r, "frontend_boot_load_time_seconds", "Frontend boot time measurement")
-	registerFrontendHistogram("frontend_boot_first_paint_time_seconds", "Frontend boot first paint")
+	registerFrontendHistogram(r, "frontend_boot_first_paint_time_seconds", "Frontend boot first paint")
-	registerFrontendHistogram("frontend_boot_first_contentful_paint_time_seconds", "Frontend boot first contentful paint")
+	registerFrontendHistogram(r, "frontend_boot_first_contentful_paint_time_seconds", "Frontend boot first contentful paint")
-	registerFrontendHistogram("frontend_boot_js_done_time_seconds", "Frontend boot initial js load")
+	registerFrontendHistogram(r, "frontend_boot_js_done_time_seconds", "Frontend boot initial js load")
-	registerFrontendHistogram("frontend_boot_css_time_seconds", "Frontend boot initial css load")
+	registerFrontendHistogram(r, "frontend_boot_css_time_seconds", "Frontend boot initial css load")
-	registerFrontendHistogram("frontend_plugins_preload_ms", "Frontend preload plugin time measurement")
+	registerFrontendHistogram(r, "frontend_plugins_preload_ms", "Frontend preload plugin time measurement")
 }

pkg/infra/metrics/metrics.go

@@ -613,7 +613,7 @@ func init() {
 }
 
 // SetBuildInformation sets the build information for this binary
-func SetBuildInformation(version, revision, branch string, buildTimestamp int64) {
+func SetBuildInformation(reg prometheus.Registerer, version, revision, branch string, buildTimestamp int64) {
 	edition := "oss"
 	if setting.IsEnterprise {
 		edition = "enterprise"
@@ -631,7 +631,7 @@ func SetBuildInformation(version, revision, branch string, buildTimestamp int64)
 		Namespace: ExporterName,
 	}, []string{"version", "revision", "branch", "goversion", "edition"})
 
-	prometheus.MustRegister(grafanaBuildVersion, grafanaBuildTimestamp)
+	reg.MustRegister(grafanaBuildVersion, grafanaBuildTimestamp)
 
 	grafanaBuildVersion.WithLabelValues(version, revision, branch, runtime.Version(), edition).Set(1)
 	grafanaBuildTimestamp.WithLabelValues(version, revision, branch, runtime.Version(), edition).Set(float64(buildTimestamp))
@@ -639,7 +639,7 @@ func SetBuildInformation(version, revision, branch string, buildTimestamp int64)
 
 // SetEnvironmentInformation exposes environment values provided by the operators as an `_info` metric.
 // If there are no environment metrics labels configured, this metric will not be exposed.
-func SetEnvironmentInformation(labels map[string]string) error {
+func SetEnvironmentInformation(reg prometheus.Registerer, labels map[string]string) error {
 	if len(labels) == 0 {
 		return nil
 	}
@@ -651,7 +651,7 @@ func SetEnvironmentInformation(labels map[string]string) error {
 		ConstLabels: labels,
 	})
 
-	prometheus.MustRegister(grafanaEnvironmentInfo)
+	reg.MustRegister(grafanaEnvironmentInfo)
 
 	grafanaEnvironmentInfo.Set(1)
 	return nil
@@ -661,8 +661,8 @@ func SetPluginBuildInformation(pluginID, pluginType, version, signatureStatus st
 	grafanaPluginBuildInfoDesc.WithLabelValues(pluginID, pluginType, version, signatureStatus).Set(1)
 }
 
-func initMetricVars() {
+func initMetricVars(reg prometheus.Registerer) {
-	prometheus.MustRegister(
+	reg.MustRegister(
 		MInstanceStart,
 		MPageStatus,
 		MApiStatus,

pkg/infra/metrics/service.go

@@ -2,11 +2,17 @@ package metrics
 
 import (
 	"context"
+	"errors"
+	"regexp"
+
+	"github.com/prometheus/client_golang/prometheus"
+	dto "github.com/prometheus/client_model/go"
+	"k8s.io/component-base/metrics/legacyregistry"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/metrics/graphitebridge"
+	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/setting"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
 var metricsLogger log.Logger = log.New("metrics")
@@ -19,12 +25,10 @@ func (lw *logWrapper) Println(v ...any) {
 	lw.logger.Info("graphite metric bridge", v...)
 }
 
-func init() {
+func ProvideService(cfg *setting.Cfg, reg prometheus.Registerer) (*InternalMetricsService, error) {
-	initMetricVars()
+	initMetricVars(reg)
-	initFrontendMetrics()
+	initFrontendMetrics(reg)
-}
 
-func ProvideService(cfg *setting.Cfg) (*InternalMetricsService, error) {
 	s := &InternalMetricsService{
 		Cfg: cfg,
 	}
@@ -55,5 +59,66 @@ func (im *InternalMetricsService) Run(ctx context.Context) error {
 	return ctx.Err()
 }
 
-func ProvideRegisterer() prometheus.Registerer        { return prometheus.DefaultRegisterer }
+func ProvideRegisterer(cfg *setting.Cfg) prometheus.Registerer {
-func ProvideRegistererForTest() prometheus.Registerer { return prometheus.NewRegistry() }
+	if cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer) {
+		return legacyregistry.Registerer()
+	}
+	return prometheus.DefaultRegisterer
+}
+
+func ProvideGatherer(cfg *setting.Cfg) prometheus.Gatherer {
+	if cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer) {
+		return newAddPrefixWrapper(legacyregistry.DefaultGatherer)
+	}
+	return prometheus.DefaultGatherer
+}
+
+func ProvideRegistererForTest() prometheus.Registerer {
+	return prometheus.NewRegistry()
+}
+
+func ProvideGathererForTest(reg prometheus.Registerer) prometheus.Gatherer {
+	// the registerer provided by ProvideRegistererForTest
+	// is a *prometheus.Registry, so it also implements prometheus.Gatherer
+	return reg.(*prometheus.Registry)
+}
+
+var _ prometheus.Gatherer = (*addPrefixWrapper)(nil)
+
+// addPrefixWrapper wraps a prometheus.Gatherer, and ensures that all metric names are prefixed with `grafana_`.
+// metrics with the prefix `grafana_` or `go_` are not modified.
+type addPrefixWrapper struct {
+	orig prometheus.Gatherer
+	reg  *regexp.Regexp
+}
+
+func newAddPrefixWrapper(orig prometheus.Gatherer) *addPrefixWrapper {
+	return &addPrefixWrapper{
+		orig: orig,
+		reg:  regexp.MustCompile("^((?:grafana_|go_).*)"),
+	}
+}
+
+func (g *addPrefixWrapper) Gather() ([]*dto.MetricFamily, error) {
+	mf, err := g.orig.Gather()
+	if err != nil {
+		return nil, err
+	}
+
+	names := make(map[string]struct{})
+
+	for i := 0; i < len(mf); i++ {
+		m := mf[i]
+		if m.Name != nil && !g.reg.MatchString(*m.Name) {
+			*m.Name = "grafana_" + *m.Name
+			// since we are modifying the name, we need to check for duplicates in the gatherer
+			if _, exists := names[*m.Name]; exists {
+				return nil, errors.New("duplicate metric name: " + *m.Name)
+			}
+		}
+		// keep track of names to detect duplicates
+		names[*m.Name] = struct{}{}
+	}
+
+	return mf, nil
+}

pkg/infra/metrics/service_test.go

@@ -0,0 +1,61 @@
+package metrics
+
+import (
+	"testing"
+
+	dto "github.com/prometheus/client_model/go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestK8sGathererWrapper_Gather(t *testing.T) {
+	orig := &mockGatherer{}
+	g := newAddPrefixWrapper(orig)
+
+	t.Run("metrics with grafana and go prefix are not modified", func(t *testing.T) {
+		originalMF := []*dto.MetricFamily{
+			{Name: strptr("grafana_metric1")},
+			{Name: strptr("metric2")},
+			{Name: strptr("go_metric1")},
+		}
+
+		orig.GatherFunc = func() ([]*dto.MetricFamily, error) {
+			return originalMF, nil
+		}
+
+		expectedMF := []*dto.MetricFamily{
+			{Name: strptr("grafana_metric1")},
+			{Name: strptr("grafana_metric2")},
+			{Name: strptr("go_metric1")},
+		}
+
+		mf, err := g.Gather()
+		require.NoError(t, err)
+		require.Equal(t, expectedMF, mf)
+	})
+
+	t.Run("duplicate metrics result in an error", func(t *testing.T) {
+		originalMF := []*dto.MetricFamily{
+			{Name: strptr("grafana_metric1")},
+			{Name: strptr("metric1")},
+		}
+
+		orig.GatherFunc = func() ([]*dto.MetricFamily, error) {
+			return originalMF, nil
+		}
+
+		_, err := g.Gather()
+		require.Error(t, err)
+	})
+}
+
+type mockGatherer struct {
+	GatherFunc func() ([]*dto.MetricFamily, error)
+}
+
+func (m *mockGatherer) Gather() ([]*dto.MetricFamily, error) {
+	return m.GatherFunc()
+}
+
+func strptr(s string) *string {
+	return &s
+}

pkg/infra/metrics/wireset.go

@@ -0,0 +1,17 @@
+package metrics
+
+import (
+	"github.com/google/wire"
+)
+
+var WireSet = wire.NewSet(
+	ProvideService,
+	ProvideRegisterer,
+	ProvideGatherer,
+)
+
+var WireSetForTest = wire.NewSet(
+	ProvideService,
+	ProvideRegistererForTest,
+	ProvideGathererForTest,
+)

pkg/server/server.go

@@ -13,6 +13,8 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
+	"github.com/prometheus/client_golang/prometheus"
+
 	"github.com/grafana/grafana/pkg/api"
 	_ "github.com/grafana/grafana/pkg/extensions"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -38,9 +40,10 @@ type Options struct {
 func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry,
 	provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry,
 	usageStatsProvidersRegistry registry.UsageStatsProvidersRegistry, statsCollectorService *statscollector.Service,
+	promReg prometheus.Registerer,
 ) (*Server, error) {
 	statsCollectorService.RegisterProviders(usageStatsProvidersRegistry.GetServices())
-	s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider)
+	s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider, promReg)
 	if err != nil {
 		return nil, err
 	}
@@ -54,11 +57,13 @@ func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistr
 
 func newServer(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry,
 	provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry,
+	promReg prometheus.Registerer,
 ) (*Server, error) {
 	rootCtx, shutdownFn := context.WithCancel(context.Background())
 	childRoutines, childCtx := errgroup.WithContext(rootCtx)
 
 	s := &Server{
+		promReg:             promReg,
 		context:             childCtx,
 		childRoutines:       childRoutines,
 		HTTPServer:          httpServer,
@@ -101,6 +106,7 @@ type Server struct {
 	HTTPServer          *api.HTTPServer
 	roleRegistry        accesscontrol.RoleRegistry
 	provisioningService provisioning.ProvisioningService
+	promReg             prometheus.Registerer
 }
 
 // Init initializes the server and its services.
@@ -117,7 +123,7 @@ func (s *Server) Init() error {
 		return err
 	}
 
-	if err := metrics.SetEnvironmentInformation(s.cfg.MetricsGrafanaEnvironmentInfo); err != nil {
+	if err := metrics.SetEnvironmentInformation(s.promReg, s.cfg.MetricsGrafanaEnvironmentInfo); err != nil {
 		return err
 	}
 

pkg/server/server_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/registry"
@@ -48,7 +49,7 @@ func (s *testService) IsDisabled() bool {
 
 func testServer(t *testing.T, services ...registry.BackgroundService) *Server {
 	t.Helper()
-	s, err := newServer(Options{}, setting.NewCfg(), nil, &acimpl.Service{}, nil, backgroundsvcs.NewBackgroundServiceRegistry(services...))
+	s, err := newServer(Options{}, setting.NewCfg(), nil, &acimpl.Service{}, nil, backgroundsvcs.NewBackgroundServiceRegistry(services...), prometheus.NewRegistry())
 	require.NoError(t, err)
 	// Required to skip configuration initialization that causes
 	// DI errors in this test.

pkg/server/wire.go

@@ -254,7 +254,6 @@ var wireBasicSet = wire.NewSet(
 	notifications.ProvideSmtpService,
 	tracing.ProvideService,
 	wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)),
-	metrics.ProvideService,
 	testdatasource.ProvideService,
 	ldapapi.ProvideService,
 	opentsdb.ProvideService,
@@ -389,7 +388,7 @@ var wireBasicSet = wire.NewSet(
 
 var wireSet = wire.NewSet(
 	wireBasicSet,
-	metrics.ProvideRegisterer,
+	metrics.WireSet,
 	sqlstore.ProvideService,
 	ngmetrics.ProvideService,
 	wire.Bind(new(notifications.Service), new(*notifications.NotificationService)),
@@ -404,6 +403,7 @@ var wireSet = wire.NewSet(
 var wireCLISet = wire.NewSet(
 	NewRunner,
 	wireBasicSet,
+	metrics.WireSet,
 	sqlstore.ProvideService,
 	ngmetrics.ProvideService,
 	wire.Bind(new(notifications.Service), new(*notifications.NotificationService)),
@@ -418,7 +418,7 @@ var wireCLISet = wire.NewSet(
 var wireTestSet = wire.NewSet(
 	wireBasicSet,
 	ProvideTestEnv,
-	metrics.ProvideRegistererForTest,
+	metrics.WireSetForTest,
 	sqlstore.ProvideServiceForTests,
 	ngmetrics.ProvideServiceForTest,
 	notifications.MockNotificationService,

pkg/server/wireexts_oss.go

@@ -7,6 +7,7 @@ package server
 import (
 	"github.com/google/wire"
 
+	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/plugins"
 	"github.com/grafana/grafana/pkg/plugins/manager"
 	"github.com/grafana/grafana/pkg/registry"
@@ -123,6 +124,7 @@ var wireExtsTestSet = wire.NewSet(
 var wireExtsBaseCLISet = wire.NewSet(
 	NewModuleRunner,
 
+	metrics.WireSet,
 	featuremgmt.ProvideManagerService,
 	featuremgmt.ProvideToggles,
 	hooks.ProvideService,