From ce512862f7815ee861ef4ab78ba4159a5d0677e9 Mon Sep 17 00:00:00 2001 From: Stephanie Hingtgen Date: Wed, 8 Jan 2025 05:57:52 -0700 Subject: [PATCH] Stats: remove dependency on dashboards and folders (#98653) --- .../statscollector/concurrent_users_test.go | 7 +- pkg/services/stats/statsimpl/stats.go | 177 +++++++++++++++--- pkg/services/stats/statsimpl/stats_test.go | 55 ++++-- 3 files changed, 200 insertions(+), 39 deletions(-) diff --git a/pkg/infra/usagestats/statscollector/concurrent_users_test.go b/pkg/infra/usagestats/statscollector/concurrent_users_test.go index decb1642657..4d89fb51a4a 100644 --- a/pkg/infra/usagestats/statscollector/concurrent_users_test.go +++ b/pkg/infra/usagestats/statscollector/concurrent_users_test.go @@ -12,6 +12,9 @@ import ( "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/folder/foldertest" + "github.com/grafana/grafana/pkg/services/org/orgtest" "github.com/grafana/grafana/pkg/services/stats/statsimpl" "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tests/testsuite" @@ -25,7 +28,7 @@ func TestMain(m *testing.M) { func TestConcurrentUsersMetrics(t *testing.T) { sqlStore, cfg := db.InitTestDBWithCfg(t) - statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore) + statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore, &dashboards.FakeDashboardService{}, &foldertest.FakeService{}, &orgtest.FakeOrgService{}) s := createService(t, cfg, sqlStore, statsService) createConcurrentTokens(t, sqlStore) @@ -43,7 +46,7 @@ func TestConcurrentUsersMetrics(t *testing.T) { func TestConcurrentUsersStats(t *testing.T) { sqlStore, cfg := db.InitTestDBWithCfg(t) - statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore) + statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore, &dashboards.FakeDashboardService{}, &foldertest.FakeService{}, &orgtest.FakeOrgService{}) s := createService(t, cfg, sqlStore, statsService) createConcurrentTokens(t, sqlStore) diff --git a/pkg/services/stats/statsimpl/stats.go b/pkg/services/stats/statsimpl/stats.go index e2139c80b57..6b27a239813 100644 --- a/pkg/services/stats/statsimpl/stats.go +++ b/pkg/services/stats/statsimpl/stats.go @@ -6,7 +6,11 @@ import ( "strconv" "time" + "github.com/grafana/authlib/claims" + "github.com/grafana/grafana/pkg/apimachinery/identity" "github.com/grafana/grafana/pkg/infra/db" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/folder" "github.com/grafana/grafana/pkg/services/libraryelements/model" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/sqlstore/migrator" @@ -17,13 +21,96 @@ import ( const activeUserTimeLimit = time.Hour * 24 * 30 const dailyActiveUserTimeLimit = time.Hour * 24 -func ProvideService(cfg *setting.Cfg, db db.DB) stats.Service { - return &sqlStatsService{cfg: cfg, db: db} +func ProvideService(cfg *setting.Cfg, db db.DB, dashSvc dashboards.DashboardService, folderSvc folder.Service, orgSvc org.Service) stats.Service { + return &sqlStatsService{ + cfg: cfg, + db: db, + folderSvc: folderSvc, + dashSvc: dashSvc, + orgSvc: orgSvc, + } } type sqlStatsService struct { - db db.DB - cfg *setting.Cfg + db db.DB + cfg *setting.Cfg + dashSvc dashboards.DashboardService + folderSvc folder.Service + orgSvc org.Service +} + +type dashboardStats struct { + count int + bytesTotal int + bytesMax int +} + +func (ss *sqlStatsService) collectDashboardStats(ctx context.Context, orgs []*org.OrgDTO, calculateByteSize bool) (dashboardStats, error) { + stats := dashboardStats{ + count: 0, + bytesTotal: 0, + bytesMax: 0, + } + + for _, org := range orgs { + ctx = identity.WithRequester(ctx, getStatsRequester(org.ID)) + dashs, err := ss.dashSvc.GetAllDashboardsByOrgId(ctx, org.ID) + if err != nil { + return stats, err + } + stats.count += len(dashs) + + // only calculate bytes if needed + if calculateByteSize { + for _, dash := range dashs { + b, err := dash.Data.ToDB() + if err != nil { + return stats, err + } + stats.bytesTotal += len(b) + + if len(b) > stats.bytesMax { + stats.bytesMax = len(b) + } + } + } + } + + return stats, nil +} + +func (ss *sqlStatsService) getTagCount(ctx context.Context, orgs []*org.OrgDTO) (int64, error) { + total := 0 + for _, org := range orgs { + ctx = identity.WithRequester(ctx, getStatsRequester(org.ID)) + tags, err := ss.dashSvc.GetDashboardTags(ctx, &dashboards.GetDashboardTagsQuery{ + OrgID: org.ID, + }) + if err != nil { + return 0, err + } + total += len(tags) + } + + return int64(total), nil +} + +func (ss *sqlStatsService) getFolderCount(ctx context.Context, orgs []*org.OrgDTO) (int64, error) { + total := 0 + for _, org := range orgs { + backgroundUser := getStatsRequester(org.ID) + ctx = identity.WithRequester(ctx, backgroundUser) + folders, err := ss.folderSvc.GetFolders(ctx, folder.GetFoldersQuery{ + OrgID: org.ID, + SignedInUser: backgroundUser, + }) + if err != nil { + return 0, err + } + + total += len(folders) + } + return int64(total), nil } func (ss *sqlStatsService) GetAlertNotifiersUsageStats(ctx context.Context, query *stats.GetAlertNotifierUsageStatsQuery) (result []*stats.NotifierUsageStats, err error) { @@ -67,7 +154,6 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS sb := &db.SQLBuilder{} sb.Write("SELECT ") sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` WHERE ` + notServiceAccount(dialect) + `) AS users,`) - sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`) sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`) sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("star") + `) AS stars,`) sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("playlist") + `) AS playlists,`) @@ -87,11 +173,6 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` WHERE `+ notServiceAccount(dialect)+` AND last_seen_at > ?) AS monthly_active_users,`, monthlyActiveUserDeadlineDate) - sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboards,`, dialect.BooleanStr(false)) - sb.Write(`(SELECT SUM(LENGTH(data)) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboard_bytes_total,`, dialect.BooleanStr(false)) - sb.Write(`(SELECT MAX(LENGTH(data)) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboard_bytes_max,`, dialect.BooleanStr(false)) - sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS folders,`, dialect.BooleanStr(true)) - sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`) sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`) sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_version") + `) AS dashboard_versions,`) @@ -124,6 +205,30 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS return nil }) + if err != nil { + return result, err + } + + orgs, err := ss.orgSvc.Search(ctx, &org.SearchOrgsQuery{}) + if err != nil { + return result, err + } + result.Orgs = int64(len(orgs)) + + // for services in unified storage, get the stats through the service rather than the db directly + dashStats, err := ss.collectDashboardStats(ctx, orgs, true) + if err != nil { + return result, err + } + result.DashboardBytesMax = int64(dashStats.bytesMax) + result.DashboardBytesTotal = int64(dashStats.bytesTotal) + result.Dashboards = int64(dashStats.count) + + folderCount, err := ss.getFolderCount(ctx, orgs) + if err != nil { + return result, err + } + result.Folders = folderCount return result, err } @@ -161,22 +266,10 @@ func (ss *sqlStatsService) GetAdminStats(ctx context.Context, query *stats.GetAd } var rawSQL = `SELECT - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("org") + ` - ) AS orgs, - ( - SELECT COUNT(*) - FROM ` + dialect.Quote("dashboard") + `WHERE is_folder=` + dialect.BooleanStr(false) + ` - ) AS dashboards, ( SELECT COUNT(*) FROM ` + dialect.Quote("dashboard_snapshot") + ` ) AS snapshots, - ( - SELECT COUNT( DISTINCT ( ` + dialect.Quote("term") + ` )) - FROM ` + dialect.Quote("dashboard_tag") + ` - ) AS tags, ( SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + ` @@ -225,6 +318,29 @@ func (ss *sqlStatsService) GetAdminStats(ctx context.Context, query *stats.GetAd result = &stats return nil }) + if err != nil { + return result, err + } + + orgs, err := ss.orgSvc.Search(ctx, &org.SearchOrgsQuery{}) + if err != nil { + return result, err + } + result.Orgs = int64(len(orgs)) + + // for services in unified storage, get the stats through the service rather than the db directly + dashStats, err := ss.collectDashboardStats(ctx, orgs, false) + if err != nil { + return result, err + } + result.Dashboards = int64(dashStats.count) + + tagCount, err := ss.getTagCount(ctx, orgs) + if err != nil { + return result, err + } + result.Tags = tagCount + return result, err } @@ -343,3 +459,20 @@ func addToStats(base stats.UserStats, role org.RoleType, count int64) stats.User return base } + +func getStatsRequester(orgId int64) *identity.StaticRequester { + return &identity.StaticRequester{ + Type: claims.TypeServiceAccount, + UserID: 1, + OrgID: orgId, + Name: "admin", + Login: "admin", + OrgRole: identity.RoleAdmin, + IsGrafanaAdmin: true, + Permissions: map[int64]map[string][]string{ + orgId: { + "*": {"*"}, + }, + }, + } +} diff --git a/pkg/services/stats/statsimpl/stats_test.go b/pkg/services/stats/statsimpl/stats_test.go index e07c059bf9d..3b3951ed27b 100644 --- a/pkg/services/stats/statsimpl/stats_test.go +++ b/pkg/services/stats/statsimpl/stats_test.go @@ -6,13 +6,18 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/components/simplejson" "github.com/grafana/grafana/pkg/infra/db" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/services/correlations" "github.com/grafana/grafana/pkg/services/correlations/correlationstest" + "github.com/grafana/grafana/pkg/services/dashboards" + "github.com/grafana/grafana/pkg/services/folder" + "github.com/grafana/grafana/pkg/services/folder/foldertest" "github.com/grafana/grafana/pkg/services/org" "github.com/grafana/grafana/pkg/services/org/orgimpl" "github.com/grafana/grafana/pkg/services/quota/quotatest" @@ -33,8 +38,30 @@ func TestIntegrationStatsDataAccess(t *testing.T) { t.Skip("skipping integration test") } db, cfg := db.InitTestDBWithCfg(t) - statsService := &sqlStatsService{db: db} - populateDB(t, db, cfg) + orgSvc := populateDB(t, db, cfg) + dashSvc := &dashboards.FakeDashboardService{} + emptyJson := simplejson.New() + emptyJsonBytes, err := emptyJson.ToDB() + require.NoError(t, err) + largerJson := simplejson.NewFromAny(map[string]string{"key": "value"}) + largerJsonBytes, err := largerJson.ToDB() + require.NoError(t, err) + dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(1)).Return([]*dashboards.Dashboard{{Data: largerJson}, {Data: emptyJson}}, nil) + dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(2)).Return([]*dashboards.Dashboard{}, nil) + dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(3)).Return([]*dashboards.Dashboard{}, nil) + dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 1}).Return([]*dashboards.DashboardTagCloudItem{{Term: "test"}}, nil) + dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 2}).Return([]*dashboards.DashboardTagCloudItem{}, nil) + dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 3}).Return([]*dashboards.DashboardTagCloudItem{}, nil) + + folderService := &foldertest.FakeService{} + folderService.ExpectedFolders = []*folder.Folder{{ID: 1}, {ID: 2}, {ID: 3}} + + statsService := &sqlStatsService{ + db: db, + dashSvc: dashSvc, + orgSvc: orgSvc, + folderSvc: folderService, + } t.Run("Get system stats should not results in error", func(t *testing.T) { query := stats.GetSystemStatsQuery{} @@ -48,6 +75,11 @@ func TestIntegrationStatsDataAccess(t *testing.T) { assert.Equal(t, int64(0), result.LibraryVariables) assert.Equal(t, int64(0), result.APIKeys) assert.Equal(t, int64(2), result.Correlations) + assert.Equal(t, int64(3), result.Orgs) + assert.Equal(t, int64(2), result.Dashboards) + assert.Equal(t, int64(9), result.Folders) // will return 3 folders for each org + assert.Equal(t, int64(len(largerJsonBytes)+len(emptyJsonBytes)), result.DashboardBytesTotal) + assert.Equal(t, int64(len(largerJsonBytes)), result.DashboardBytesMax) assert.NotNil(t, result.DatabaseCreatedTime) assert.Equal(t, db.GetDialect().DriverName(), result.DatabaseDriver) }) @@ -78,12 +110,15 @@ func TestIntegrationStatsDataAccess(t *testing.T) { t.Run("Get admin stats should not result in error", func(t *testing.T) { query := stats.GetAdminStatsQuery{} - _, err := statsService.GetAdminStats(context.Background(), &query) + stats, err := statsService.GetAdminStats(context.Background(), &query) assert.NoError(t, err) + assert.Equal(t, int64(1), stats.Tags) + assert.Equal(t, int64(2), stats.Dashboards) + assert.Equal(t, int64(3), stats.Orgs) }) } -func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) { +func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) org.Service { t.Helper() orgService, _ := orgimpl.ProvideService(db, cfg, quotatest.New(false, nil)) @@ -151,16 +186,6 @@ func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) { } err = orgService.AddOrgUser(context.Background(), cmd) require.NoError(t, err) -} -func TestIntegration_GetAdminStats(t *testing.T) { - if testing.Short() { - t.Skip("skipping integration test") - } - db, cfg := db.InitTestDBWithCfg(t) - statsService := ProvideService(cfg, db) - - query := stats.GetAdminStatsQuery{} - _, err := statsService.GetAdminStats(context.Background(), &query) - require.NoError(t, err) + return orgService }