diff --git a/pkg/apiserver/builder/common.go b/pkg/apiserver/builder/common.go index d4a0d661c0e..e8741100bd5 100644 --- a/pkg/apiserver/builder/common.go +++ b/pkg/apiserver/builder/common.go @@ -4,6 +4,7 @@ import ( "net/http" grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" + "github.com/prometheus/client_golang/prometheus" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/runtime/serializer" @@ -29,6 +30,7 @@ type APIGroupBuilder interface { codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, desiredMode grafanarest.DualWriterMode, + reg prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) // Get OpenAPI definitions diff --git a/pkg/apiserver/builder/helper.go b/pkg/apiserver/builder/helper.go index ed223a7667e..1fe320aceb5 100644 --- a/pkg/apiserver/builder/helper.go +++ b/pkg/apiserver/builder/helper.go @@ -25,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/apiserver/endpoints/filters" "github.com/grafana/grafana/pkg/services/apiserver/options" + "github.com/prometheus/client_golang/prometheus" ) // TODO: this is a temporary hack to make rest.Connecter work with resource level routes @@ -128,6 +129,7 @@ func InstallAPIs( optsGetter generic.RESTOptionsGetter, builders []APIGroupBuilder, storageOpts *options.StorageOptions, + reg prometheus.Registerer, ) error { // dual writing is only enabled when the storage type is not legacy. // this is needed to support setting a default RESTOptionsGetter for new APIs that don't @@ -136,7 +138,7 @@ func InstallAPIs( for _, b := range builders { mode := b.GetDesiredDualWriterMode(dualWriteEnabled, storageOpts.DualWriterDesiredModes) - g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode) + g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode, reg) if err != nil { return err } diff --git a/pkg/apiserver/rest/dualwriter.go b/pkg/apiserver/rest/dualwriter.go index 9031229b999..f979c932d6b 100644 --- a/pkg/apiserver/rest/dualwriter.go +++ b/pkg/apiserver/rest/dualwriter.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/grafana/grafana/pkg/infra/kvstore" + "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apiserver/pkg/registry/rest" @@ -95,25 +96,28 @@ const ( Mode4 ) +// TODO: make this function private as there should only be one public way of setting the dual writing mode // NewDualWriter returns a new DualWriter. -func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage) DualWriter { +func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage, reg prometheus.Registerer) DualWriter { + metrics := &dualWriterMetrics{} + metrics.init(reg) switch mode { // It is not possible to initialize a mode 0 dual writer. Mode 0 represents // writing to legacy storage without `unifiedStorage` enabled. case Mode1: // read and write only from legacy storage - return newDualWriterMode1(legacy, storage) + return newDualWriterMode1(legacy, storage, metrics) case Mode2: // write to both, read from storage but use legacy as backup - return newDualWriterMode2(legacy, storage) + return newDualWriterMode2(legacy, storage, metrics) case Mode3: // write to both, read from storage only - return newDualWriterMode3(legacy, storage) + return newDualWriterMode3(legacy, storage, metrics) case Mode4: // read and write only from storage - return newDualWriterMode4(legacy, storage) + return newDualWriterMode4(legacy, storage, metrics) default: - return newDualWriterMode1(legacy, storage) + return newDualWriterMode1(legacy, storage, metrics) } } @@ -142,6 +146,7 @@ func SetDualWritingMode( storage Storage, entity string, desiredMode DualWriterMode, + reg prometheus.Registerer, ) (DualWriter, error) { toMode := map[string]DualWriterMode{ // It is not possible to initialize a mode 0 dual writer. Mode 0 represents @@ -200,5 +205,5 @@ func SetDualWritingMode( // #TODO add support for other combinations of desired and current modes - return NewDualWriter(currentMode, legacy, storage), nil + return NewDualWriter(currentMode, legacy, storage, reg), nil } diff --git a/pkg/apiserver/rest/dualwriter_mode1.go b/pkg/apiserver/rest/dualwriter_mode1.go index 15f79f5b711..03b9e226ed2 100644 --- a/pkg/apiserver/rest/dualwriter_mode1.go +++ b/pkg/apiserver/rest/dualwriter_mode1.go @@ -24,10 +24,8 @@ const mode1Str = "1" // NewDualWriterMode1 returns a new DualWriter in mode 1. // Mode 1 represents writing to and reading from LegacyStorage. -func newDualWriterMode1(legacy LegacyStorage, storage Storage) *DualWriterMode1 { - metrics := &dualWriterMetrics{} - metrics.init() - return &DualWriterMode1{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode1"), dualWriterMetrics: metrics} +func newDualWriterMode1(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode1 { + return &DualWriterMode1{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode1"), dualWriterMetrics: dwm} } // Mode returns the mode of the dual writer. diff --git a/pkg/apiserver/rest/dualwriter_mode1_test.go b/pkg/apiserver/rest/dualwriter_mode1_test.go index d248e631373..998d683bedf 100644 --- a/pkg/apiserver/rest/dualwriter_mode1_test.go +++ b/pkg/apiserver/rest/dualwriter_mode1_test.go @@ -5,6 +5,7 @@ import ( "errors" "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "k8s.io/apimachinery/pkg/api/meta" @@ -22,6 +23,8 @@ var failingObj = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta var exampleList = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, Items: []example.Pod{*exampleObj}} var anotherList = &example.PodList{Items: []example.Pod{*anotherObj}} +var p = prometheus.NewRegistry() + func TestMode1_Create(t *testing.T) { type testCase struct { input runtime.Object @@ -68,7 +71,7 @@ func TestMode1_Create(t *testing.T) { tt.setupStorageFn(m) } - dw := NewDualWriter(Mode1, ls, us) + dw := NewDualWriter(Mode1, ls, us, p) obj, err := dw.Create(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.CreateOptions{}) @@ -131,7 +134,8 @@ func TestMode1_Get(t *testing.T) { tt.setupStorageFn(m, tt.input) } - dw := NewDualWriter(Mode1, ls, us) + p := prometheus.NewRegistry() + dw := NewDualWriter(Mode1, ls, us, p) obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{}) @@ -182,7 +186,7 @@ func TestMode1_List(t *testing.T) { tt.setupStorageFn(m) } - dw := NewDualWriter(Mode1, ls, us) + dw := NewDualWriter(Mode1, ls, us, p) _, err := dw.List(context.Background(), &metainternalversion.ListOptions{}) @@ -237,7 +241,7 @@ func TestMode1_Delete(t *testing.T) { tt.setupStorageFn(m, tt.input) } - dw := NewDualWriter(Mode1, ls, us) + dw := NewDualWriter(Mode1, ls, us, p) obj, _, err := dw.Delete(context.Background(), tt.input, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{}) @@ -296,7 +300,7 @@ func TestMode1_DeleteCollection(t *testing.T) { tt.setupStorageFn(m, tt.input) } - dw := NewDualWriter(Mode1, ls, us) + dw := NewDualWriter(Mode1, ls, us, p) obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, tt.input, &metainternalversion.ListOptions{}) @@ -372,7 +376,7 @@ func TestMode1_Update(t *testing.T) { tt.setupGetFn(m, tt.input) } - dw := NewDualWriter(Mode1, ls, us) + dw := NewDualWriter(Mode1, ls, us, p) obj, _, err := dw.Update(context.Background(), tt.input, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{}) diff --git a/pkg/apiserver/rest/dualwriter_mode2.go b/pkg/apiserver/rest/dualwriter_mode2.go index 8ff4a653cc9..36051dd7eaa 100644 --- a/pkg/apiserver/rest/dualwriter_mode2.go +++ b/pkg/apiserver/rest/dualwriter_mode2.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/grafana/grafana/pkg/apimachinery/utils" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" @@ -13,8 +14,6 @@ import ( "k8s.io/apimachinery/pkg/selection" "k8s.io/apiserver/pkg/registry/rest" "k8s.io/klog/v2" - - "github.com/grafana/grafana/pkg/apimachinery/utils" ) type DualWriterMode2 struct { @@ -28,10 +27,8 @@ const mode2Str = "2" // NewDualWriterMode2 returns a new DualWriter in mode 2. // Mode 2 represents writing to LegacyStorage and Storage and reading from LegacyStorage. -func newDualWriterMode2(legacy LegacyStorage, storage Storage) *DualWriterMode2 { - metrics := &dualWriterMetrics{} - metrics.init() - return &DualWriterMode2{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode2"), dualWriterMetrics: metrics} +func newDualWriterMode2(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode2 { + return &DualWriterMode2{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode2"), dualWriterMetrics: dwm} } // Mode returns the mode of the dual writer. diff --git a/pkg/apiserver/rest/dualwriter_mode2_test.go b/pkg/apiserver/rest/dualwriter_mode2_test.go index ef028fbafa1..6b0a10e1537 100644 --- a/pkg/apiserver/rest/dualwriter_mode2_test.go +++ b/pkg/apiserver/rest/dualwriter_mode2_test.go @@ -67,7 +67,7 @@ func TestMode2_Create(t *testing.T) { tt.setupStorageFn(m, tt.input) } - dw := NewDualWriter(Mode2, ls, us) + dw := NewDualWriter(Mode2, ls, us, p) obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{}) @@ -143,7 +143,7 @@ func TestMode2_Get(t *testing.T) { tt.setupStorageFn(m, tt.input) } - dw := NewDualWriter(Mode2, ls, us) + dw := NewDualWriter(Mode2, ls, us, p) obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{}) @@ -196,7 +196,7 @@ func TestMode2_List(t *testing.T) { tt.setupStorageFn(m) } - dw := NewDualWriter(Mode2, ls, us) + dw := NewDualWriter(Mode2, ls, us, p) obj, err := dw.List(context.Background(), &metainternalversion.ListOptions{}) @@ -289,7 +289,7 @@ func TestMode2_Delete(t *testing.T) { tt.setupStorageFn(m, tt.input) } - dw := NewDualWriter(Mode2, ls, us) + dw := NewDualWriter(Mode2, ls, us, p) obj, _, err := dw.Delete(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{}) @@ -361,7 +361,7 @@ func TestMode2_DeleteCollection(t *testing.T) { tt.setupStorageFn(m) } - dw := NewDualWriter(Mode2, ls, us) + dw := NewDualWriter(Mode2, ls, us, p) obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: tt.input}}, &metainternalversion.ListOptions{}) @@ -469,7 +469,7 @@ func TestMode2_Update(t *testing.T) { tt.setupStorageFn(m, tt.input) } - dw := NewDualWriter(Mode2, ls, us) + dw := NewDualWriter(Mode2, ls, us, p) obj, _, err := dw.Update(context.Background(), tt.input, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{}) diff --git a/pkg/apiserver/rest/dualwriter_mode3.go b/pkg/apiserver/rest/dualwriter_mode3.go index 50f21fd39ba..f24d58627ae 100644 --- a/pkg/apiserver/rest/dualwriter_mode3.go +++ b/pkg/apiserver/rest/dualwriter_mode3.go @@ -20,10 +20,8 @@ type DualWriterMode3 struct { // newDualWriterMode3 returns a new DualWriter in mode 3. // Mode 3 represents writing to LegacyStorage and Storage and reading from Storage. -func newDualWriterMode3(legacy LegacyStorage, storage Storage) *DualWriterMode3 { - metrics := &dualWriterMetrics{} - metrics.init() - return &DualWriterMode3{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode3"), dualWriterMetrics: metrics} +func newDualWriterMode3(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode3 { + return &DualWriterMode3{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode3"), dualWriterMetrics: dwm} } // Mode returns the mode of the dual writer. diff --git a/pkg/apiserver/rest/dualwriter_mode4.go b/pkg/apiserver/rest/dualwriter_mode4.go index de15ee8d5c7..7985e69332e 100644 --- a/pkg/apiserver/rest/dualwriter_mode4.go +++ b/pkg/apiserver/rest/dualwriter_mode4.go @@ -19,10 +19,8 @@ type DualWriterMode4 struct { // newDualWriterMode4 returns a new DualWriter in mode 4. // Mode 4 represents writing and reading from Storage. -func newDualWriterMode4(legacy LegacyStorage, storage Storage) *DualWriterMode4 { - metrics := &dualWriterMetrics{} - metrics.init() - return &DualWriterMode4{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode4"), dualWriterMetrics: metrics} +func newDualWriterMode4(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode4 { + return &DualWriterMode4{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode4"), dualWriterMetrics: dwm} } // Mode returns the mode of the dual writer. diff --git a/pkg/apiserver/rest/dualwriter_test.go b/pkg/apiserver/rest/dualwriter_test.go index 10cb16644d5..6c9a8da9619 100644 --- a/pkg/apiserver/rest/dualwriter_test.go +++ b/pkg/apiserver/rest/dualwriter_test.go @@ -7,6 +7,7 @@ import ( playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1" "github.com/grafana/grafana/pkg/infra/kvstore" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" ) @@ -45,7 +46,8 @@ func TestSetDualWritingMode(t *testing.T) { kvStore := kvstore.WithNamespace(kvstore.NewFakeKVStore(), 0, "storage.dualwriting."+tt.stackID) - dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode) + p := prometheus.NewRegistry() + dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode, p) assert.NoError(t, err) assert.Equal(t, tt.expectedMode, dw.Mode()) diff --git a/pkg/apiserver/rest/metrics.go b/pkg/apiserver/rest/metrics.go index a4e72457a8f..4e5db41e0e2 100644 --- a/pkg/apiserver/rest/metrics.go +++ b/pkg/apiserver/rest/metrics.go @@ -5,6 +5,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" + "k8s.io/klog/v2" ) type dualWriterMetrics struct { @@ -37,10 +38,17 @@ var DualWriterOutcome = prometheus.NewHistogramVec(prometheus.HistogramOpts{ NativeHistogramBucketFactor: 1.1, }, []string{"mode", "name", "method"}) -func (m *dualWriterMetrics) init() { +func (m *dualWriterMetrics) init(reg prometheus.Registerer) { + log := klog.NewKlogr() m.legacy = DualWriterLegacyDuration m.storage = DualWriterStorageDuration m.outcome = DualWriterOutcome + errLegacy := reg.Register(m.legacy) + errStorage := reg.Register(m.storage) + errOutcome := reg.Register(m.outcome) + if errLegacy != nil || errStorage != nil || errOutcome != nil { + log.Info("cloud migration metrics already registered") + } } func (m *dualWriterMetrics) recordLegacyDuration(isError bool, mode string, name string, method string, startFrom time.Time) { diff --git a/pkg/cmd/grafana/apiserver/server.go b/pkg/cmd/grafana/apiserver/server.go index 8278868f451..0963337bea4 100644 --- a/pkg/cmd/grafana/apiserver/server.go +++ b/pkg/cmd/grafana/apiserver/server.go @@ -165,7 +165,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf // Install the API Group+version // #TODO figure out how to configure storage type in o.Options.StorageOptions - err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, o.Options.StorageOptions) + err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, o.Options.StorageOptions, o.Options.MetricsOptions.MetricsRegisterer) if err != nil { return err } diff --git a/pkg/registry/apis/dashboard/register.go b/pkg/registry/apis/dashboard/register.go index 731939ce1bf..fe60abed4fb 100644 --- a/pkg/registry/apis/dashboard/register.go +++ b/pkg/registry/apis/dashboard/register.go @@ -25,6 +25,7 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/provisioning" "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*DashboardsAPIBuilder)(nil) @@ -49,6 +50,7 @@ func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, accessControl accesscontrol.AccessControl, provisioning provisioning.ProvisioningService, dashStore dashboards.Store, + reg prometheus.Registerer, sql db.DB, ) *DashboardsAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { @@ -115,6 +117,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, desiredMode grafanarest.DualWriterMode, + reg prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -145,7 +148,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo( if err := store.CompleteWithOptions(options); err != nil { return nil, err } - storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store) + storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store, reg) } // Summary diff --git a/pkg/registry/apis/dashboardsnapshot/register.go b/pkg/registry/apis/dashboardsnapshot/register.go index 7d884889ad1..67db0abfa70 100644 --- a/pkg/registry/apis/dashboardsnapshot/register.go +++ b/pkg/registry/apis/dashboardsnapshot/register.go @@ -8,6 +8,7 @@ import ( "time" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" @@ -71,6 +72,7 @@ func RegisterAPIService( cfg *setting.Cfg, features featuremgmt.FeatureToggles, sql db.DB, + reg prometheus.Registerer, ) *SnapshotsAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis @@ -129,6 +131,7 @@ func (b *SnapshotsAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, _ grafanarest.DualWriterMode, + _ prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(dashboardsnapshot.GROUP, scheme, metav1.ParameterCodec, codecs) storage := map[string]rest.Storage{} diff --git a/pkg/registry/apis/datasource/register.go b/pkg/registry/apis/datasource/register.go index c1eab887823..93370ed49d7 100644 --- a/pkg/registry/apis/datasource/register.go +++ b/pkg/registry/apis/datasource/register.go @@ -7,6 +7,7 @@ import ( "time" "github.com/grafana/grafana-plugin-sdk-go/backend" + "github.com/prometheus/client_golang/prometheus" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" @@ -56,6 +57,7 @@ func RegisterAPIService( contextProvider PluginContextWrapper, pluginStore pluginstore.Store, accessControl accesscontrol.AccessControl, + reg prometheus.Registerer, ) (*DataSourceAPIBuilder, error) { // This requires devmode! if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { @@ -205,6 +207,7 @@ func (b *DataSourceAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, // pointer? _ generic.RESTOptionsGetter, _ grafanarest.DualWriterMode, + _ prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { storage := map[string]rest.Storage{} diff --git a/pkg/registry/apis/example/register.go b/pkg/registry/apis/example/register.go index 9b7da8ff1ef..edb2a90bbad 100644 --- a/pkg/registry/apis/example/register.go +++ b/pkg/registry/apis/example/register.go @@ -25,6 +25,7 @@ import ( grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*TestingAPIBuilder)(nil) @@ -41,7 +42,7 @@ func NewTestingAPIBuilder() *TestingAPIBuilder { } } -func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *TestingAPIBuilder { +func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar, reg prometheus.Registerer) *TestingAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis } @@ -92,6 +93,7 @@ func (b *TestingAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, // pointer? _ generic.RESTOptionsGetter, _ grafanarest.DualWriterMode, + _ prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { b.codecs = codecs apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(b.gv.Group, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/featuretoggle/register.go b/pkg/registry/apis/featuretoggle/register.go index 72d135d451c..0173057954a 100644 --- a/pkg/registry/apis/featuretoggle/register.go +++ b/pkg/registry/apis/featuretoggle/register.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/services/accesscontrol" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*FeatureFlagAPIBuilder)(nil) @@ -40,6 +41,7 @@ func RegisterAPIService(features *featuremgmt.FeatureManager, accessControl accesscontrol.AccessControl, apiregistration builder.APIRegistrar, cfg *setting.Cfg, + registerer prometheus.Registerer, ) *FeatureFlagAPIBuilder { builder := NewFeatureFlagAPIBuilder(features, accessControl, cfg) apiregistration.RegisterAPI(builder) @@ -89,6 +91,7 @@ func (b *FeatureFlagAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, // pointer? _ generic.RESTOptionsGetter, _ grafanarest.DualWriterMode, + _ prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/folders/register.go b/pkg/registry/apis/folders/register.go index 62eafeb7290..d658c68db49 100644 --- a/pkg/registry/apis/folders/register.go +++ b/pkg/registry/apis/folders/register.go @@ -27,6 +27,7 @@ import ( "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/folder" "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*FolderAPIBuilder)(nil) @@ -47,6 +48,7 @@ func RegisterAPIService(cfg *setting.Cfg, apiregistration builder.APIRegistrar, folderSvc folder.Service, accessControl accesscontrol.AccessControl, + registerer prometheus.Registerer, ) *FolderAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis @@ -106,6 +108,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, desiredMode grafanarest.DualWriterMode, + reg prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs) @@ -145,7 +148,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo( if err != nil { return nil, err } - storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store) + storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store, reg) } apiGroupInfo.VersionedResourcesStorageMap[v0alpha1.VERSION] = storage diff --git a/pkg/registry/apis/peakq/register.go b/pkg/registry/apis/peakq/register.go index 718c9746edf..21e1aa072d9 100644 --- a/pkg/registry/apis/peakq/register.go +++ b/pkg/registry/apis/peakq/register.go @@ -17,6 +17,7 @@ import ( "github.com/grafana/grafana/pkg/apiserver/builder" grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*PeakQAPIBuilder)(nil) @@ -28,7 +29,7 @@ func NewPeakQAPIBuilder() *PeakQAPIBuilder { return &PeakQAPIBuilder{} } -func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *PeakQAPIBuilder { +func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar, reg prometheus.Registerer) *PeakQAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis } @@ -73,6 +74,7 @@ func (b *PeakQAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, _ grafanarest.DualWriterMode, // dual write desired mode (not relevant) + _ prometheus.Registerer, // prometheus registerer ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(peakq.GROUP, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/playlist/register.go b/pkg/registry/apis/playlist/register.go index b9902946a6c..1c3666777ca 100644 --- a/pkg/registry/apis/playlist/register.go +++ b/pkg/registry/apis/playlist/register.go @@ -23,6 +23,7 @@ import ( gapiutil "github.com/grafana/grafana/pkg/services/apiserver/utils" playlistsvc "github.com/grafana/grafana/pkg/services/playlist" "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*PlaylistAPIBuilder)(nil) @@ -39,12 +40,14 @@ func RegisterAPIService(p playlistsvc.Service, apiregistration builder.APIRegistrar, cfg *setting.Cfg, kvStore kvstore.KVStore, + registerer prometheus.Registerer, ) *PlaylistAPIBuilder { builder := &PlaylistAPIBuilder{ service: p, namespacer: request.GetNamespaceMapper(cfg), gv: playlist.PlaylistResourceInfo.GroupVersion(), kvStore: kvstore.WithNamespace(kvStore, 0, "storage.dualwriting"), + // register: newMetrics(registerer), } apiregistration.RegisterAPI(builder) return builder @@ -94,6 +97,7 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, desiredMode grafanarest.DualWriterMode, + reg prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(playlist.GROUP, scheme, metav1.ParameterCodec, codecs) storage := map[string]rest.Storage{} @@ -133,7 +137,7 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo( return nil, err } - dualWriter, err := grafanarest.SetDualWritingMode(context.Background(), b.kvStore, legacyStore, store, playlist.GROUPRESOURCE, desiredMode) + dualWriter, err := grafanarest.SetDualWritingMode(context.Background(), b.kvStore, legacyStore, store, playlist.GROUPRESOURCE, desiredMode, reg) if err != nil { return nil, err } diff --git a/pkg/registry/apis/query/metrics.go b/pkg/registry/apis/query/metrics.go index e13525917e2..f3b8212f9db 100644 --- a/pkg/registry/apis/query/metrics.go +++ b/pkg/registry/apis/query/metrics.go @@ -9,15 +9,15 @@ const ( metricsNamespace = "grafana" ) -type metrics struct { +type queryMetrics struct { dsRequests *prometheus.CounterVec // older metric expressionsQuerySummary *prometheus.SummaryVec } -func newMetrics(reg prometheus.Registerer) *metrics { - m := &metrics{ +func newQueryMetrics(reg prometheus.Registerer) *queryMetrics { + m := &queryMetrics{ dsRequests: prometheus.NewCounterVec(prometheus.CounterOpts{ Namespace: metricsNamespace, Subsystem: metricsSubSystem, diff --git a/pkg/registry/apis/query/register.go b/pkg/registry/apis/query/register.go index 897f045c102..9ae0b235cc0 100644 --- a/pkg/registry/apis/query/register.go +++ b/pkg/registry/apis/query/register.go @@ -42,7 +42,7 @@ type QueryAPIBuilder struct { features featuremgmt.FeatureToggles tracer tracing.Tracer - metrics *metrics + metrics *queryMetrics parser *queryParser client DataSourceClientSupplier registry query.DataSourceApiServerRegistry @@ -81,7 +81,7 @@ func NewQueryAPIBuilder(features featuremgmt.FeatureToggles, client: client, registry: registry, parser: newQueryParser(reader, legacy, tracer), - metrics: newMetrics(registerer), + metrics: newQueryMetrics(registerer), tracer: tracer, features: features, queryTypes: queryTypes, @@ -151,6 +151,7 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, // pointer? optsGetter generic.RESTOptionsGetter, _ grafanarest.DualWriterMode, + _ prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { gv := query.SchemeGroupVersion apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/scope/register.go b/pkg/registry/apis/scope/register.go index 91215ce7a7e..f86839f8591 100644 --- a/pkg/registry/apis/scope/register.go +++ b/pkg/registry/apis/scope/register.go @@ -19,6 +19,7 @@ import ( "github.com/grafana/grafana/pkg/apiserver/builder" grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*ScopeAPIBuilder)(nil) @@ -30,7 +31,7 @@ func NewScopeAPIBuilder() *ScopeAPIBuilder { return &ScopeAPIBuilder{} } -func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *ScopeAPIBuilder { +func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar, reg prometheus.Registerer) *ScopeAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) { return nil // skip registration unless opting into experimental apis } @@ -121,6 +122,7 @@ func (b *ScopeAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, _ grafanarest.DualWriterMode, // dual write desired mode (not relevant) + _ prometheus.Registerer, // prometheus registerer ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(scope.GROUP, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/registry/apis/service/register.go b/pkg/registry/apis/service/register.go index 69b01a2707d..6635233d1fd 100644 --- a/pkg/registry/apis/service/register.go +++ b/pkg/registry/apis/service/register.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/grafana/pkg/apiserver/builder" grafanarest "github.com/grafana/grafana/pkg/apiserver/rest" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/prometheus/client_golang/prometheus" ) var _ builder.APIGroupBuilder = (*ServiceAPIBuilder)(nil) @@ -26,7 +27,7 @@ func NewServiceAPIBuilder() *ServiceAPIBuilder { return &ServiceAPIBuilder{} } -func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *ServiceAPIBuilder { +func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar, registerer prometheus.Registerer) *ServiceAPIBuilder { if !features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) { return nil // skip registration unless opting into aggregator mode } @@ -79,6 +80,7 @@ func (b *ServiceAPIBuilder) GetAPIGroupInfo( codecs serializer.CodecFactory, optsGetter generic.RESTOptionsGetter, _ grafanarest.DualWriterMode, + _ prometheus.Registerer, ) (*genericapiserver.APIGroupInfo, error) { apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(service.GROUP, scheme, metav1.ParameterCodec, codecs) diff --git a/pkg/services/apiserver/aggregator/aggregator.go b/pkg/services/apiserver/aggregator/aggregator.go index 7107b2d77f5..8c97479c17b 100644 --- a/pkg/services/apiserver/aggregator/aggregator.go +++ b/pkg/services/apiserver/aggregator/aggregator.go @@ -24,6 +24,7 @@ import ( servicev0alpha1 "github.com/grafana/grafana/pkg/apis/service/v0alpha1" "github.com/grafana/grafana/pkg/registry/apis/service" + "github.com/prometheus/client_golang/prometheus" "gopkg.in/yaml.v3" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" @@ -172,7 +173,7 @@ func CreateAggregatorConfig(commandOptions *options.Options, sharedConfig generi return NewConfig(aggregatorConfig, sharedInformerFactory, []builder.APIGroupBuilder{serviceAPIBuilder}, remoteServicesConfig), nil } -func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.DelegationTarget) (*aggregatorapiserver.APIAggregator, error) { +func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.DelegationTarget, reg prometheus.Registerer) (*aggregatorapiserver.APIAggregator, error) { aggregatorConfig := config.KubeAggregatorConfig sharedInformerFactory := config.Informers remoteServicesConfig := config.RemoteServicesConfig @@ -285,7 +286,13 @@ func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.D }) for _, b := range config.Builders { - serviceAPIGroupInfo, err := b.GetAPIGroupInfo(aggregatorscheme.Scheme, aggregatorscheme.Codecs, aggregatorConfig.GenericConfig.RESTOptionsGetter, grafanarest.Mode0) + serviceAPIGroupInfo, err := b.GetAPIGroupInfo( + aggregatorscheme.Scheme, + aggregatorscheme.Codecs, + aggregatorConfig.GenericConfig.RESTOptionsGetter, + grafanarest.Mode0, + reg, + ) if err != nil { return nil, err } diff --git a/pkg/services/apiserver/service.go b/pkg/services/apiserver/service.go index 2a61456d206..339b52bfd05 100644 --- a/pkg/services/apiserver/service.go +++ b/pkg/services/apiserver/service.go @@ -7,6 +7,7 @@ import ( "path" "github.com/grafana/dskit/services" + "github.com/prometheus/client_golang/prometheus" "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -24,6 +25,7 @@ import ( filestorage "github.com/grafana/grafana/pkg/apiserver/storage/file" "github.com/grafana/grafana/pkg/infra/appcontext" "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/middleware" "github.com/grafana/grafana/pkg/modules" @@ -109,6 +111,7 @@ type service struct { builders []builder.APIGroupBuilder tracing *tracing.TracingService + metrics prometheus.Registerer authorizer *authorizer.GrafanaAuthorizer } @@ -131,6 +134,7 @@ func ProvideService( authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService), tracing: tracing, db: db, // For Unified storage + metrics: metrics.ProvideRegisterer(), } // This will be used when running as a dskit service @@ -314,7 +318,7 @@ func (s *service) start(ctx context.Context) error { } // Install the API group+version - err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions) + err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions, s.metrics) if err != nil { return err } @@ -324,7 +328,7 @@ func (s *service) start(ctx context.Context) error { var runningServer *genericapiserver.GenericAPIServer if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) { - runningServer, err = s.startAggregator(transport, serverConfig, server) + runningServer, err = s.startAggregator(transport, serverConfig, server, s.metrics) if err != nil { return err } @@ -374,6 +378,7 @@ func (s *service) startAggregator( transport *roundTripperFunc, serverConfig *genericapiserver.RecommendedConfig, server *genericapiserver.GenericAPIServer, + reg prometheus.Registerer, ) (*genericapiserver.GenericAPIServer, error) { namespaceMapper := request.GetNamespaceMapper(s.cfg) @@ -382,7 +387,7 @@ func (s *service) startAggregator( return nil, err } - aggregatorServer, err := aggregator.CreateAggregatorServer(aggregatorConfig, server) + aggregatorServer, err := aggregator.CreateAggregatorServer(aggregatorConfig, server, reg) if err != nil { return nil, err } diff --git a/pkg/services/apiserver/standalone/options/metrics.go b/pkg/services/apiserver/standalone/options/metrics.go index 4a213bd4606..11f6bdc9391 100644 --- a/pkg/services/apiserver/standalone/options/metrics.go +++ b/pkg/services/apiserver/standalone/options/metrics.go @@ -15,7 +15,7 @@ type MetricsOptions struct { MetricsRegisterer prometheus.Registerer } -func NewMetrcicsOptions(logger log.Logger) *MetricsOptions { +func NewMetricsOptions(logger log.Logger) *MetricsOptions { return &MetricsOptions{ logger: logger, } diff --git a/pkg/services/apiserver/standalone/options/options.go b/pkg/services/apiserver/standalone/options/options.go index 76085fc1558..7d1f25efdcd 100644 --- a/pkg/services/apiserver/standalone/options/options.go +++ b/pkg/services/apiserver/standalone/options/options.go @@ -25,7 +25,7 @@ func New(logger log.Logger, codec runtime.Codec) *Options { ExtraOptions: options.NewExtraOptions(), RecommendedOptions: options.NewRecommendedOptions(codec), TracingOptions: NewTracingOptions(logger), - MetricsOptions: NewMetrcicsOptions(logger), + MetricsOptions: NewMetricsOptions(logger), ServerRunOptions: genericoptions.NewServerRunOptions(), StorageOptions: options.NewStorageOptions(), }