IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Stats: remove dependency on dashboards and folders (#98653)

Browse Source
This commit is contained in:
Stephanie Hingtgen 2025-01-08 05:57:52 -07:00 committed by GitHub
parent 338a41f178
commit ce512862f7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 200 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/infra/usagestats/statscollector/concurrent_users_test.go
View File

@ -12,6 +12,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/folder/foldertest"
"github.com/grafana/grafana/pkg/services/org/orgtest"
"github.com/grafana/grafana/pkg/services/stats/statsimpl"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/tests/testsuite"
@ -25,7 +28,7 @@ func TestMain(m *testing.M) {
func TestConcurrentUsersMetrics(t *testing.T) {
sqlStore, cfg := db.InitTestDBWithCfg(t)
statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore)
statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore, &dashboards.FakeDashboardService{}, &foldertest.FakeService{}, &orgtest.FakeOrgService{})
s := createService(t, cfg, sqlStore, statsService)
createConcurrentTokens(t, sqlStore)
@ -43,7 +46,7 @@ func TestConcurrentUsersMetrics(t *testing.T) {
func TestConcurrentUsersStats(t *testing.T) {
sqlStore, cfg := db.InitTestDBWithCfg(t)
statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore)
statsService := statsimpl.ProvideService(&setting.Cfg{}, sqlStore, &dashboards.FakeDashboardService{}, &foldertest.FakeService{}, &orgtest.FakeOrgService{})
s := createService(t, cfg, sqlStore, statsService)
createConcurrentTokens(t, sqlStore)

177
pkg/services/stats/statsimpl/stats.go
View File

@ -6,7 +6,11 @@ import (
"strconv"
"time"
"github.com/grafana/authlib/claims"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/libraryelements/model"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
@ -17,13 +21,96 @@ import (
const activeUserTimeLimit = time.Hour * 24 * 30
const dailyActiveUserTimeLimit = time.Hour * 24
func ProvideService(cfg *setting.Cfg, db db.DB) stats.Service {
return &sqlStatsService{cfg: cfg, db: db}
func ProvideService(cfg *setting.Cfg, db db.DB, dashSvc dashboards.DashboardService, folderSvc folder.Service, orgSvc org.Service) stats.Service {
return &sqlStatsService{
cfg: cfg,
db: db,
folderSvc: folderSvc,
dashSvc: dashSvc,
orgSvc: orgSvc,
}
}
type sqlStatsService struct {
db db.DB
cfg *setting.Cfg
db db.DB
cfg *setting.Cfg
dashSvc dashboards.DashboardService
folderSvc folder.Service
orgSvc org.Service
}
type dashboardStats struct {
count int
bytesTotal int
bytesMax int
}
func (ss *sqlStatsService) collectDashboardStats(ctx context.Context, orgs []*org.OrgDTO, calculateByteSize bool) (dashboardStats, error) {
stats := dashboardStats{
count: 0,
bytesTotal: 0,
bytesMax: 0,
}
for _, org := range orgs {
ctx = identity.WithRequester(ctx, getStatsRequester(org.ID))
dashs, err := ss.dashSvc.GetAllDashboardsByOrgId(ctx, org.ID)
if err != nil {
return stats, err
}
stats.count += len(dashs)
// only calculate bytes if needed
if calculateByteSize {
for _, dash := range dashs {
b, err := dash.Data.ToDB()
if err != nil {
return stats, err
}
stats.bytesTotal += len(b)
if len(b) > stats.bytesMax {
stats.bytesMax = len(b)
}
}
}
}
return stats, nil
}
func (ss *sqlStatsService) getTagCount(ctx context.Context, orgs []*org.OrgDTO) (int64, error) {
total := 0
for _, org := range orgs {
ctx = identity.WithRequester(ctx, getStatsRequester(org.ID))
tags, err := ss.dashSvc.GetDashboardTags(ctx, &dashboards.GetDashboardTagsQuery{
OrgID: org.ID,
})
if err != nil {
return 0, err
}
total += len(tags)
}
return int64(total), nil
}
func (ss *sqlStatsService) getFolderCount(ctx context.Context, orgs []*org.OrgDTO) (int64, error) {
total := 0
for _, org := range orgs {
backgroundUser := getStatsRequester(org.ID)
ctx = identity.WithRequester(ctx, backgroundUser)
folders, err := ss.folderSvc.GetFolders(ctx, folder.GetFoldersQuery{
OrgID: org.ID,
SignedInUser: backgroundUser,
})
if err != nil {
return 0, err
}
total += len(folders)
}
return int64(total), nil
}
func (ss *sqlStatsService) GetAlertNotifiersUsageStats(ctx context.Context, query *stats.GetAlertNotifierUsageStatsQuery) (result []*stats.NotifierUsageStats, err error) {
@ -67,7 +154,6 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
sb := &db.SQLBuilder{}
sb.Write("SELECT ")
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("user") + ` WHERE ` + notServiceAccount(dialect) + `) AS users,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("org") + `) AS orgs,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("data_source") + `) AS datasources,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("star") + `) AS stars,`)
sb.Write(`(SELECT COUNT(*) FROM ` + dialect.Quote("playlist") + `) AS playlists,`)
@ -87,11 +173,6 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
sb.Write(`(SELECT COUNT(*) FROM `+dialect.Quote("user")+` WHERE `+
notServiceAccount(dialect)+` AND last_seen_at > ?) AS monthly_active_users,`, monthlyActiveUserDeadlineDate)
sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboards,`, dialect.BooleanStr(false))
sb.Write(`(SELECT SUM(LENGTH(data)) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboard_bytes_total,`, dialect.BooleanStr(false))
sb.Write(`(SELECT MAX(LENGTH(data)) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS dashboard_bytes_max,`, dialect.BooleanStr(false))
sb.Write(`(SELECT COUNT(id) FROM `+dialect.Quote("dashboard")+` WHERE is_folder = ?) AS folders,`, dialect.BooleanStr(true))
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_provisioning") + `) AS provisioned_dashboards,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_snapshot") + `) AS snapshots,`)
sb.Write(`(SELECT COUNT(id) FROM ` + dialect.Quote("dashboard_version") + `) AS dashboard_versions,`)
@ -124,6 +205,30 @@ func (ss *sqlStatsService) GetSystemStats(ctx context.Context, query *stats.GetS
return nil
})
if err != nil {
return result, err
}
orgs, err := ss.orgSvc.Search(ctx, &org.SearchOrgsQuery{})
if err != nil {
return result, err
}
result.Orgs = int64(len(orgs))
// for services in unified storage, get the stats through the service rather than the db directly
dashStats, err := ss.collectDashboardStats(ctx, orgs, true)
if err != nil {
return result, err
}
result.DashboardBytesMax = int64(dashStats.bytesMax)
result.DashboardBytesTotal = int64(dashStats.bytesTotal)
result.Dashboards = int64(dashStats.count)
folderCount, err := ss.getFolderCount(ctx, orgs)
if err != nil {
return result, err
}
result.Folders = folderCount
return result, err
}
@ -161,22 +266,10 @@ func (ss *sqlStatsService) GetAdminStats(ctx context.Context, query *stats.GetAd
}
var rawSQL = `SELECT
(
SELECT COUNT(*)
FROM ` + dialect.Quote("org") + `
) AS orgs,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("dashboard") + `WHERE is_folder=` + dialect.BooleanStr(false) + `
) AS dashboards,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("dashboard_snapshot") + `
) AS snapshots,
(
SELECT COUNT( DISTINCT ( ` + dialect.Quote("term") + ` ))
FROM ` + dialect.Quote("dashboard_tag") + `
) AS tags,
(
SELECT COUNT(*)
FROM ` + dialect.Quote("data_source") + `
@ -225,6 +318,29 @@ func (ss *sqlStatsService) GetAdminStats(ctx context.Context, query *stats.GetAd
result = &stats
return nil
})
if err != nil {
return result, err
}
orgs, err := ss.orgSvc.Search(ctx, &org.SearchOrgsQuery{})
if err != nil {
return result, err
}
result.Orgs = int64(len(orgs))
// for services in unified storage, get the stats through the service rather than the db directly
dashStats, err := ss.collectDashboardStats(ctx, orgs, false)
if err != nil {
return result, err
}
result.Dashboards = int64(dashStats.count)
tagCount, err := ss.getTagCount(ctx, orgs)
if err != nil {
return result, err
}
result.Tags = tagCount
return result, err
}
@ -343,3 +459,20 @@ func addToStats(base stats.UserStats, role org.RoleType, count int64) stats.User
return base
}
func getStatsRequester(orgId int64) *identity.StaticRequester {
return &identity.StaticRequester{
Type: claims.TypeServiceAccount,
UserID: 1,
OrgID: orgId,
Name: "admin",
Login: "admin",
OrgRole: identity.RoleAdmin,
IsGrafanaAdmin: true,
Permissions: map[int64]map[string][]string{
orgId: {
"*": {"*"},
},
},
}
}

55
pkg/services/stats/statsimpl/stats_test.go
View File

@ -6,13 +6,18 @@ import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/correlations"
"github.com/grafana/grafana/pkg/services/correlations/correlationstest"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/folder/foldertest"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/org/orgimpl"
"github.com/grafana/grafana/pkg/services/quota/quotatest"
@ -33,8 +38,30 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
t.Skip("skipping integration test")
}
db, cfg := db.InitTestDBWithCfg(t)
statsService := &sqlStatsService{db: db}
populateDB(t, db, cfg)
orgSvc := populateDB(t, db, cfg)
dashSvc := &dashboards.FakeDashboardService{}
emptyJson := simplejson.New()
emptyJsonBytes, err := emptyJson.ToDB()
require.NoError(t, err)
largerJson := simplejson.NewFromAny(map[string]string{"key": "value"})
largerJsonBytes, err := largerJson.ToDB()
require.NoError(t, err)
dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(1)).Return([]*dashboards.Dashboard{{Data: largerJson}, {Data: emptyJson}}, nil)
dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(2)).Return([]*dashboards.Dashboard{}, nil)
dashSvc.On("GetAllDashboardsByOrgId", mock.Anything, int64(3)).Return([]*dashboards.Dashboard{}, nil)
dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 1}).Return([]*dashboards.DashboardTagCloudItem{{Term: "test"}}, nil)
dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 2}).Return([]*dashboards.DashboardTagCloudItem{}, nil)
dashSvc.On("GetDashboardTags", mock.Anything, &dashboards.GetDashboardTagsQuery{OrgID: 3}).Return([]*dashboards.DashboardTagCloudItem{}, nil)
folderService := &foldertest.FakeService{}
folderService.ExpectedFolders = []*folder.Folder{{ID: 1}, {ID: 2}, {ID: 3}}
statsService := &sqlStatsService{
db: db,
dashSvc: dashSvc,
orgSvc: orgSvc,
folderSvc: folderService,
}
t.Run("Get system stats should not results in error", func(t *testing.T) {
query := stats.GetSystemStatsQuery{}
@ -48,6 +75,11 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
assert.Equal(t, int64(0), result.LibraryVariables)
assert.Equal(t, int64(0), result.APIKeys)
assert.Equal(t, int64(2), result.Correlations)
assert.Equal(t, int64(3), result.Orgs)
assert.Equal(t, int64(2), result.Dashboards)
assert.Equal(t, int64(9), result.Folders) // will return 3 folders for each org
assert.Equal(t, int64(len(largerJsonBytes)+len(emptyJsonBytes)), result.DashboardBytesTotal)
assert.Equal(t, int64(len(largerJsonBytes)), result.DashboardBytesMax)
assert.NotNil(t, result.DatabaseCreatedTime)
assert.Equal(t, db.GetDialect().DriverName(), result.DatabaseDriver)
})
@ -78,12 +110,15 @@ func TestIntegrationStatsDataAccess(t *testing.T) {
t.Run("Get admin stats should not result in error", func(t *testing.T) {
query := stats.GetAdminStatsQuery{}
_, err := statsService.GetAdminStats(context.Background(), &query)
stats, err := statsService.GetAdminStats(context.Background(), &query)
assert.NoError(t, err)
assert.Equal(t, int64(1), stats.Tags)
assert.Equal(t, int64(2), stats.Dashboards)
assert.Equal(t, int64(3), stats.Orgs)
})
}
func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) {
func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) org.Service {
t.Helper()
orgService, _ := orgimpl.ProvideService(db, cfg, quotatest.New(false, nil))
@ -151,16 +186,6 @@ func populateDB(t *testing.T, db db.DB, cfg *setting.Cfg) {
}
err = orgService.AddOrgUser(context.Background(), cmd)
require.NoError(t, err)
}
func TestIntegration_GetAdminStats(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
db, cfg := db.InitTestDBWithCfg(t)
statsService := ProvideService(cfg, db)
query := stats.GetAdminStatsQuery{}
_, err := statsService.GetAdminStats(context.Background(), &query)
require.NoError(t, err)
return orgService
}