Stats: remove dependency on dashboards and folders (#98653)

pkg/infra/usagestats/statscollector/concurrent_users_test.go

@@ -12,6 +12,9 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/services/dashboards"
+	"github.com/grafana/grafana/pkg/services/folder/foldertest"
+	"github.com/grafana/grafana/pkg/services/org/orgtest"
 	"github.com/grafana/grafana/pkg/services/stats/statsimpl"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tests/testsuite"
@@ -25,7 +28,7 @@ func TestMain(m *testing.M) {
 
 func TestConcurrentUsersMetrics(t *testing.T) {
 	sqlStore, cfg := db.InitTestDBWithCfg(t)
-	statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore)
+	statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore, &dashboards.FakeDashboardService{}, &foldertest.FakeService{}, &orgtest.FakeOrgService{})
 	s := createService(t, cfg, sqlStore, statsService)
 
 	createConcurrentTokens(t, sqlStore)
@@ -43,7 +46,7 @@ func TestConcurrentUsersMetrics(t *testing.T) {
 
 func TestConcurrentUsersStats(t *testing.T) {
 	sqlStore, cfg := db.InitTestDBWithCfg(t)
-	statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore)
+	statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore, &dashboards.FakeDashboardService{}, &foldertest.FakeService{}, &orgtest.FakeOrgService{})
 	s := createService(t, cfg, sqlStore, statsService)
 
 	createConcurrentTokens(t, sqlStore)

pkg/services/stats/statsimpl/stats.go

@@ -6,7 +6,11 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/grafana/authlib/claims"
+	"github.com/grafana/grafana/pkg/apimachinery/identity"
 	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/services/dashboards"
+	"github.com/grafana/grafana/pkg/services/folder"
 	"github.com/grafana/grafana/pkg/services/libraryelements/model"
 	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
@@ -17,13 +21,96 @@ import (
 const activeUserTimeLimit = time.Hour * 24 * 30
 const dailyActiveUserTimeLimit = time.Hour * 24
 
-func ProvideService(cfg *setting.Cfg, db db.DB) stats.Service {
+func ProvideService(cfg *setting.Cfg, db db.DB, dashSvc dashboards.DashboardService, folderSvc folder.Service, orgSvc org.Service) stats.Service {
-	return &sqlStatsService{cfg: cfg, db: db}
+	return &sqlStatsService{
+		cfg:       cfg,
+		db:        db,
+		folderSvc: folderSvc,
+		dashSvc:   dashSvc,
+		orgSvc:    orgSvc,
+	}
 }
 
 type sqlStatsService struct {
-	db  db.DB
+	db        db.DB
-	cfg *setting.Cfg
+	cfg       *setting.Cfg
+	dashSvc   dashboards.DashboardService
+	folderSvc folder.Service
+	orgSvc    org.Service
+}
+
+type dashboardStats struct {
+	count      int
+	bytesTotal int
+	bytesMax   int
+}
+
+func (ss *sqlStatsService) collectDashboardStats(ctx context.Context, orgs []*org.OrgDTO, calculateByteSize bool) (dashboardStats, error) {
+	stats := dashboardStats{
+		count:      0,
+		bytesTotal: 0,
+		bytesMax:   0,
+	}
+
+	for _, org := range orgs {
+		ctx = identity.WithRequester(ctx, getStatsRequester(org.ID))
+		dashs, err := ss.dashSvc.GetAllDashboardsByOrgId(ctx, org.ID)
+		if err != nil {
+			return stats, err
+		}
+		stats.count += len(dashs)
+
+		// only calculate bytes if needed
+		if calculateByteSize {
+			for _, dash := range dashs {
+				b, err := dash.Data.ToDB()
+				if err != nil {
+					return stats, err
+				}
+				stats.bytesTotal += len(b)
+
+				if len(b) > stats.bytesMax {
+					stats.bytesMax = len(b)
+				}
+			}
+		}
+	}
+
+	return stats, nil
+}
+
+func (ss *sqlStatsService) getTagCount(ctx context.Context, orgs []*org.OrgDTO) (int64, error) {
+	total := 0
+	for _, org := range orgs {
+		ctx = identity.WithRequester(ctx, getStatsRequester(org.ID))
+		tags, err := ss.dashSvc.GetDashboardTags(ctx, &dashboards.GetDashboardTagsQuery{
+			OrgID: org.ID,
+		})
+		if err != nil {
+			return 0, err
+		}
+		total += len(tags)
+	}
+
+	return int64(total), nil
+}
+
+func (ss *sqlStatsService) getFolderCount(ctx context.Context, orgs []*org.OrgDTO) (int64, error) {
+	total := 0
+	for _, org := range orgs {
+		backgroundUser := getStatsRequester(org.ID)
+		ctx = identity.WithRequester(ctx, backgroundUser)
+		folders, err := ss.folderSvc.GetFolders(ctx, folder.GetFoldersQuery{
+			OrgID:        org.ID,
+			SignedInUser: backgroundUser,
+		})
+		if err != nil {
+			return 0, err
+		}
+
+		total += len(folders)
+	}
+	return int64(total), nil
 }
 
 func (ss *sqlStatsService) GetAlertNotifiersUsageStats(ctx context.Context, query *stats.GetAlertNotifierUsageStatsQuery) (result []*stats.NotifierUsageStats, err error) {
@@ -67,7 +154,6 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
 		sb := &db.SQLBuilder{}
 		sb.Write("SELECT ")
 		sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` WHERE ` + notServiceAccount(dialect) + `) AS users,`)
-		sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`)
 		sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`)
 		sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("star") + `) AS stars,`)
 		sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("playlist") + `) AS playlists,`)
@@ -87,11 +173,6 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
 		sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` WHERE `+
 			notServiceAccount(dialect)+` AND last_seen_at > ?) AS monthly_active_users,`, monthlyActiveUserDeadlineDate)
 
-		sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboards,`, dialect.BooleanStr(false))
-		sb.Write(`(SELECT SUM(LENGTH(data)) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboard_bytes_total,`, dialect.BooleanStr(false))
-		sb.Write(`(SELECT MAX(LENGTH(data)) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboard_bytes_max,`, dialect.BooleanStr(false))
-		sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS folders,`, dialect.BooleanStr(true))
-
 		sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`)
 		sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`)
 		sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_version") + `) AS dashboard_versions,`)
@@ -124,6 +205,30 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
 
 		return nil
 	})
+	if err != nil {
+		return result, err
+	}
+
+	orgs, err := ss.orgSvc.Search(ctx, &org.SearchOrgsQuery{})
+	if err != nil {
+		return result, err
+	}
+	result.Orgs = int64(len(orgs))
+
+	// for services in unified storage, get the stats through the service rather than the db directly
+	dashStats, err := ss.collectDashboardStats(ctx, orgs, true)
+	if err != nil {
+		return result, err
+	}
+	result.DashboardBytesMax = int64(dashStats.bytesMax)
+	result.DashboardBytesTotal = int64(dashStats.bytesTotal)
+	result.Dashboards = int64(dashStats.count)
+
+	folderCount, err := ss.getFolderCount(ctx, orgs)
+	if err != nil {
+		return result, err
+	}
+	result.Folders = folderCount
 
 	return result, err
 }
@@ -161,22 +266,10 @@ func (ss *sqlStatsService) GetAdminStats(ctx context.Context, query *stats.GetAd
 		}
 
 		var rawSQL = `SELECT
-		(
-			SELECT COUNT(*)
-			FROM ` + dialect.Quote("org") + `
-		) AS orgs,
-		(
-			SELECT COUNT(*)
-			FROM ` + dialect.Quote("dashboard") + `WHERE is_folder=` + dialect.BooleanStr(false) + `
-		) AS dashboards,
 		(
 			SELECT COUNT(*)
 			FROM ` + dialect.Quote("dashboard_snapshot") + `
 		) AS snapshots,
-		(
-			SELECT COUNT( DISTINCT ( ` + dialect.Quote("term") + ` ))
-			FROM ` + dialect.Quote("dashboard_tag") + `
-		) AS tags,
 		(
 			SELECT COUNT(*)
 			FROM ` + dialect.Quote("data_source") + `
@@ -225,6 +318,29 @@ func (ss *sqlStatsService) GetAdminStats(ctx context.Context, query *stats.GetAd
 		result = &stats
 		return nil
 	})
+	if err != nil {
+		return result, err
+	}
+
+	orgs, err := ss.orgSvc.Search(ctx, &org.SearchOrgsQuery{})
+	if err != nil {
+		return result, err
+	}
+	result.Orgs = int64(len(orgs))
+
+	// for services in unified storage, get the stats through the service rather than the db directly
+	dashStats, err := ss.collectDashboardStats(ctx, orgs, false)
+	if err != nil {
+		return result, err
+	}
+	result.Dashboards = int64(dashStats.count)
+
+	tagCount, err := ss.getTagCount(ctx, orgs)
+	if err != nil {
+		return result, err
+	}
+	result.Tags = tagCount
+
 	return result, err
 }
 
@@ -343,3 +459,20 @@ func addToStats(base stats.UserStats, role org.RoleType, count int64) stats.User
 
 	return base
 }
+
+func getStatsRequester(orgId int64) *identity.StaticRequester {
+	return &identity.StaticRequester{
+		Type:           claims.TypeServiceAccount,
+		UserID:         1,
+		OrgID:          orgId,
+		Name:           "admin",
+		Login:          "admin",
+		OrgRole:        identity.RoleAdmin,
+		IsGrafanaAdmin: true,
+		Permissions: map[int64]map[string][]string{
+			orgId: {
+				"*": {"*"},
+			},
+		},
+	}
+}

pkg/services/stats/statsimpl/stats_test.go

@@ -6,13 +6,18 @@ import (
 	"testing"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 
 	"github.com/grafana/grafana/pkg/bus"
+	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/correlations"
 	"github.com/grafana/grafana/pkg/services/correlations/correlationstest"
+	"github.com/grafana/grafana/pkg/services/dashboards"
+	"github.com/grafana/grafana/pkg/services/folder"
+	"github.com/grafana/grafana/pkg/services/folder/foldertest"
 	"github.com/grafana/grafana/pkg/services/org"
 	"github.com/grafana/grafana/pkg/services/org/orgimpl"
 	"github.com/grafana/grafana/pkg/services/quota/quotatest"
@@ -33,8 +38,30 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
 		t.Skip("skipping integration test")
 	}
 	db, cfg := db.InitTestDBWithCfg(t)
-	statsService := &sqlStatsService{db: db}
+	orgSvc := populateDB(t, db, cfg)
-	populateDB(t, db, cfg)
+	dashSvc := &dashboards.FakeDashboardService{}
+	emptyJson := simplejson.New()
+	emptyJsonBytes, err := emptyJson.ToDB()
+	require.NoError(t, err)
+	largerJson := simplejson.NewFromAny(map[string]string{"key": "value"})
+	largerJsonBytes, err := largerJson.ToDB()
+	require.NoError(t, err)
+	dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(1)).Return([]*dashboards.Dashboard{{Data: largerJson}, {Data: emptyJson}}, nil)
+	dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(2)).Return([]*dashboards.Dashboard{}, nil)
+	dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(3)).Return([]*dashboards.Dashboard{}, nil)
+	dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 1}).Return([]*dashboards.DashboardTagCloudItem{{Term: "test"}}, nil)
+	dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 2}).Return([]*dashboards.DashboardTagCloudItem{}, nil)
+	dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 3}).Return([]*dashboards.DashboardTagCloudItem{}, nil)
+
+	folderService := &foldertest.FakeService{}
+	folderService.ExpectedFolders = []*folder.Folder{{ID: 1}, {ID: 2}, {ID: 3}}
+
+	statsService := &sqlStatsService{
+		db:        db,
+		dashSvc:   dashSvc,
+		orgSvc:    orgSvc,
+		folderSvc: folderService,
+	}
 
 	t.Run("Get system stats should not results in error", func(t *testing.T) {
 		query := stats.GetSystemStatsQuery{}
@@ -48,6 +75,11 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
 		assert.Equal(t, int64(0), result.LibraryVariables)
 		assert.Equal(t, int64(0), result.APIKeys)
 		assert.Equal(t, int64(2), result.Correlations)
+		assert.Equal(t, int64(3), result.Orgs)
+		assert.Equal(t, int64(2), result.Dashboards)
+		assert.Equal(t, int64(9), result.Folders) // will return 3 folders for each org
+		assert.Equal(t, int64(len(largerJsonBytes)+len(emptyJsonBytes)), result.DashboardBytesTotal)
+		assert.Equal(t, int64(len(largerJsonBytes)), result.DashboardBytesMax)
 		assert.NotNil(t, result.DatabaseCreatedTime)
 		assert.Equal(t, db.GetDialect().DriverName(), result.DatabaseDriver)
 	})
@@ -78,12 +110,15 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
 
 	t.Run("Get admin stats should not result in error", func(t *testing.T) {
 		query := stats.GetAdminStatsQuery{}
-		_, err := statsService.GetAdminStats(context.Background(), &query)
+		stats, err := statsService.GetAdminStats(context.Background(), &query)
 		assert.NoError(t, err)
+		assert.Equal(t, int64(1), stats.Tags)
+		assert.Equal(t, int64(2), stats.Dashboards)
+		assert.Equal(t, int64(3), stats.Orgs)
 	})
 }
 
-func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) {
+func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) org.Service {
 	t.Helper()
 
 	orgService, _ := orgimpl.ProvideService(db, cfg, quotatest.New(false, nil))
@@ -151,16 +186,6 @@ func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) {
 	}
 	err = orgService.AddOrgUser(context.Background(), cmd)
 	require.NoError(t, err)
-}
 
-func TestIntegration_GetAdminStats(t *testing.T) {
+	return orgService
-	if testing.Short() {
-		t.Skip("skipping integration test")
-	}
-	db, cfg := db.InitTestDBWithCfg(t)
-	statsService := ProvideService(cfg, db)
-
-	query := stats.GetAdminStatsQuery{}
-	_, err := statsService.GetAdminStats(context.Background(), &query)
-	require.NoError(t, err)
 }