Chore: Add Usage stats providers registry (#48357)

* add usage stats providers

* migrate thumbnails service to `registry.ProvidesUsageStats`

* snake -> camel

* lint fix

* migrate accesscontrol to `registry.ProvidesUsageStats`

* add accesscontrol to usage stats providers registry

* fix test

* added a note about errors
This commit is contained in:
Artur Wierzbicki 2022-04-28 13:06:49 +04:00 committed by GitHub
parent ff23f78b51
commit 736be74128
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 123 additions and 39 deletions

View File

@ -17,7 +17,6 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/remotecache"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/accesscontrol/database"
@ -395,8 +394,7 @@ func setupHTTPServerWithCfgDb(t *testing.T, useFakeAccessControl, enableAccessCo
require.NoError(t, err)
hs.teamPermissionsService = teamPermissionService
} else {
ac, errInitAc := ossaccesscontrol.ProvideService(hs.Features, &usagestats.UsageStatsMock{T: t},
database.ProvideService(db), routing.NewRouteRegister())
ac, errInitAc := ossaccesscontrol.ProvideService(hs.Features, database.ProvideService(db), routing.NewRouteRegister())
require.NoError(t, errInitAc)
hs.AccessControl = ac
// Perform role registration

View File

@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/login/social"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/sqlstore"
@ -34,6 +35,7 @@ type Service struct {
startTime time.Time
concurrentUserStatsCache memoConcurrentUserStats
promFlavorCache memoPrometheusFlavor
usageStatProviders []registry.ProvidesUsageStats
}
func ProvideService(
@ -65,6 +67,12 @@ func ProvideService(
return s
}
// RegisterProviders is called only once - during Grafana start up
func (s *Service) RegisterProviders(usageStatProviders []registry.ProvidesUsageStats) {
s.log.Info("registering usage stat providers", "usageStatsProvidersLen", len(usageStatProviders))
s.usageStatProviders = usageStatProviders
}
func (s *Service) Run(ctx context.Context) error {
s.updateTotalStats(ctx)
updateStatsTicker := time.NewTicker(time.Minute * 30)
@ -280,6 +288,13 @@ func (s *Service) collect(ctx context.Context) (map[string]interface{}, error) {
m[k] = v
}
for _, usageStatProvider := range s.usageStatProviders {
stats := usageStatProvider.GetUsageStats(ctx)
for k, v := range stats {
m[k] = v
}
}
return m, nil
}

View File

@ -8,6 +8,7 @@ import (
"time"
sdkhttpclient "github.com/grafana/grafana-plugin-sdk-go/backend/httpclient"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/infra/httpclient"
@ -78,6 +79,34 @@ func TestTotalStatsUpdate(t *testing.T) {
}
}
var _ registry.ProvidesUsageStats = (*dummyUsageStatProvider)(nil)
type dummyUsageStatProvider struct {
stats map[string]interface{}
}
func (d dummyUsageStatProvider) GetUsageStats(ctx context.Context) map[string]interface{} {
return d.stats
}
func TestUsageStatsProviders(t *testing.T) {
provider1 := &dummyUsageStatProvider{stats: map[string]interface{}{"my_stat_1": "val1", "my_stat_2": "val2"}}
provider2 := &dummyUsageStatProvider{stats: map[string]interface{}{"my_stat_x": "valx", "my_stat_z": "valz"}}
store := mockstore.NewSQLStoreMock()
mockSystemStats(store)
s := createService(t, setting.NewCfg(), store)
s.RegisterProviders([]registry.ProvidesUsageStats{provider1, provider2})
m, err := s.collect(context.Background())
require.NoError(t, err, "Expected no error")
assert.Equal(t, "val1", m["my_stat_1"])
assert.Equal(t, "val2", m["my_stat_2"])
assert.Equal(t, "valx", m["my_stat_x"])
assert.Equal(t, "valz", m["my_stat_z"])
}
func TestFeatureUsageStats(t *testing.T) {
store := mockstore.NewSQLStoreMock()
mockSystemStats(store)

View File

@ -29,6 +29,18 @@ type BackgroundService interface {
Run(ctx context.Context) error
}
// UsageStatsProvidersRegistry provides services sharing their usage stats
type UsageStatsProvidersRegistry interface {
GetServices() []ProvidesUsageStats
}
// ProvidesUsageStats is an interface for services that share their usage stats
type ProvidesUsageStats interface {
// GetUsageStats is called on a schedule by the UsageStatsService
// Any errors occurring during usage stats collection should be collected and logged within the provider.
GetUsageStats(ctx context.Context) map[string]interface{}
}
// DatabaseMigrator allows the caller to add migrations to
// the migrator passed as argument
type DatabaseMigrator interface {

View File

@ -12,6 +12,7 @@ import (
"strconv"
"sync"
"github.com/grafana/grafana/pkg/infra/usagestats/statscollector"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/api"
@ -41,7 +42,9 @@ type Options struct {
// New returns a new instance of Server.
func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry,
provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry,
usageStatsProvidersRegistry registry.UsageStatsProvidersRegistry, statsCollectorService *statscollector.Service,
) (*Server, error) {
statsCollectorService.RegisterProviders(usageStatsProvidersRegistry.GetServices())
s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider)
if err != nil {
return nil, err

View File

@ -0,0 +1,29 @@
package usagestatssvcs
import (
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/thumbs"
)
func ProvideUsageStatsProvidersRegistry(
thumbsService thumbs.Service,
accesscontrol accesscontrol.AccessControl,
) *UsageStatsProvidersRegistry {
return NewUsageStatsProvidersRegistry(
thumbsService,
accesscontrol,
)
}
type UsageStatsProvidersRegistry struct {
Services []registry.ProvidesUsageStats
}
func NewUsageStatsProvidersRegistry(services ...registry.ProvidesUsageStats) *UsageStatsProvidersRegistry {
return &UsageStatsProvidersRegistry{services}
}
func (r *UsageStatsProvidersRegistry) GetServices() []registry.ProvidesUsageStats {
return r.Services
}

View File

@ -11,6 +11,7 @@ import (
"github.com/grafana/grafana/pkg/plugins/manager/signature"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/server/backgroundsvcs"
"github.com/grafana/grafana/pkg/server/usagestatssvcs"
"github.com/grafana/grafana/pkg/services/accesscontrol"
acdb "github.com/grafana/grafana/pkg/services/accesscontrol/database"
"github.com/grafana/grafana/pkg/services/accesscontrol/ossaccesscontrol"
@ -82,6 +83,8 @@ var wireExtsBasicSet = wire.NewSet(
wire.Bind(new(permissions.DatasourcePermissionsService), new(*permissions.OSSDatasourcePermissionsService)),
ossaccesscontrol.ProvidePermissionsServices,
wire.Bind(new(accesscontrol.PermissionsServices), new(*ossaccesscontrol.PermissionsServices)),
usagestatssvcs.ProvideUsageStatsProvidersRegistry,
wire.Bind(new(registry.UsageStatsProvidersRegistry), new(*usagestatssvcs.UsageStatsProvidersRegistry)),
)
var wireExtsSet = wire.NewSet(

View File

@ -6,6 +6,7 @@ import (
"strings"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/registry"
)
type Options struct {
@ -13,6 +14,8 @@ type Options struct {
}
type AccessControl interface {
registry.ProvidesUsageStats
// Evaluate evaluates access to the given resources.
Evaluate(ctx context.Context, user *models.SignedInUser, evaluator Evaluator) (bool, error)

View File

@ -65,6 +65,10 @@ func New() *Mock {
return mock
}
func (m Mock) GetUsageStats(ctx context.Context) map[string]interface{} {
return make(map[string]interface{})
}
func (m Mock) WithPermissions(permissions []*accesscontrol.Permission) *Mock {
m.permissions = permissions
return &m

View File

@ -7,7 +7,6 @@ import (
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/accesscontrol/api"
@ -15,11 +14,10 @@ import (
"github.com/prometheus/client_golang/prometheus"
)
func ProvideService(features featuremgmt.FeatureToggles, usageStats usagestats.Service,
func ProvideService(features featuremgmt.FeatureToggles,
provider accesscontrol.PermissionsProvider, routeRegister routing.RouteRegister) (*OSSAccessControlService, error) {
var errDeclareRoles error
s := ProvideOSSAccessControl(features, provider)
s.registerUsageMetrics(usageStats)
if !s.IsDisabled() {
api := api.AccessControlAPI{
RouteRegister: routeRegister,
@ -33,7 +31,6 @@ func ProvideService(features featuremgmt.FeatureToggles, usageStats usagestats.S
return s, errDeclareRoles
}
// ProvideOSSAccessControl creates an oss implementation of access control without usage stats registration
func ProvideOSSAccessControl(features featuremgmt.FeatureToggles, provider accesscontrol.PermissionsProvider) *OSSAccessControlService {
s := &OSSAccessControlService{
features: features,
@ -63,12 +60,10 @@ func (ac *OSSAccessControlService) IsDisabled() bool {
return !ac.features.IsEnabled(featuremgmt.FlagAccesscontrol)
}
func (ac *OSSAccessControlService) registerUsageMetrics(usageStats usagestats.Service) {
usageStats.RegisterMetricsFunc(func(context.Context) (map[string]interface{}, error) {
return map[string]interface{}{
"stats.oss.accesscontrol.enabled.count": ac.getUsageMetrics(),
}, nil
})
func (ac *OSSAccessControlService) GetUsageStats(_ context.Context) map[string]interface{} {
return map[string]interface{}{
"stats.oss.accesscontrol.enabled.count": ac.getUsageMetrics(),
}
}
func (ac *OSSAccessControlService) getUsageMetrics() interface{} {

View File

@ -9,7 +9,6 @@ import (
"github.com/grafana/grafana/pkg/api/routing"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/accesscontrol/database"
@ -134,19 +133,13 @@ func TestUsageMetrics(t *testing.T) {
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
usagestatsmock := &usagestats.UsageStatsMock{T: t}
_, errInitAc := ProvideService(
s, errInitAc := ProvideService(
featuremgmt.WithFeatures("accesscontrol", tt.enabled),
usagestatsmock,
database.ProvideService(sqlstore.InitTestDB(t)),
routing.NewRouteRegister(),
)
require.NoError(t, errInitAc)
report, err := usagestatsmock.GetUsageReport(context.Background())
assert.Nil(t, err)
assert.Equal(t, tt.expectedValue, report.Metrics["stats.oss.accesscontrol.enabled.count"])
assert.Equal(t, tt.expectedValue, s.GetUsageStats(context.Background())["stats.oss.accesscontrol.enabled.count"])
})
}
}

View File

@ -11,6 +11,10 @@ import (
// When the feature flag is not enabled we just implement a dummy service
type dummyService struct{}
func (ds *dummyService) GetUsageStats(ctx context.Context) map[string]interface{} {
return make(map[string]interface{})
}
func (ds *dummyService) GetImage(c *models.ReqContext) {
c.JSON(400, map[string]string{"error": "invalid size"})
}

View File

@ -12,8 +12,8 @@ import (
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/serverlock"
"github.com/grafana/grafana/pkg/infra/usagestats"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/registry"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/guardian"
"github.com/grafana/grafana/pkg/services/live"
@ -25,6 +25,7 @@ import (
)
type Service interface {
registry.ProvidesUsageStats
Run(ctx context.Context) error
Enabled() bool
GetImage(c *models.ReqContext)
@ -50,7 +51,6 @@ type thumbService struct {
store sqlstore.Store
crawlLockServiceActionName string
log log.Logger
usageStatsService usagestats.Service
canRunCrawler bool
settings setting.DashboardPreviewsSettings
}
@ -65,7 +65,7 @@ type crawlerScheduleOptions struct {
auth CrawlerAuth
}
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore, authSetupService CrawlerAuthSetupService, usageStatsService usagestats.Service) Service {
func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore, authSetupService CrawlerAuthSetupService) Service {
if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) {
return &dummyService{}
}
@ -81,7 +81,6 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS
canRunCrawler = false
}
t := &thumbService{
usageStatsService: usageStatsService,
renderingService: renderService,
renderer: newSimpleCrawler(renderService, gl, thumbnailRepo, cfg, cfg.DashboardPreviews),
thumbnailRepo: thumbnailRepo,
@ -104,26 +103,23 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS
},
}
t.registerUsageStats()
return t
}
func (hs *thumbService) registerUsageStats() {
hs.usageStatsService.RegisterMetricsFunc(func(ctx context.Context) (map[string]interface{}, error) {
s := hs.getDashboardPreviewsSetupSettings(ctx)
func (hs *thumbService) GetUsageStats(ctx context.Context) map[string]interface{} {
s := hs.getDashboardPreviewsSetupSettings(ctx)
stats := make(map[string]interface{})
stats := make(map[string]interface{})
if s.SystemRequirements.Met {
stats["stats.dashboard_previews.system_req_met.count"] = 1
}
if s.SystemRequirements.Met {
stats["stats.dashboard_previews.system_req_met.count"] = 1
}
if s.ThumbnailsExist {
stats["stats.dashboard_previews.thumbnails_exist.count"] = 1
}
if s.ThumbnailsExist {
stats["stats.dashboard_previews.thumbnails_exist.count"] = 1
}
return stats, nil
})
return stats
}
func (hs *thumbService) Enabled() bool {