IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2026-08-18 17:15:08 -05:00
Code Issues Packages Projects Releases Wiki Activity

Split Delete expired dashboard version store (#49610)

Browse Source 
* Split Delete expired dashboard version store

* Add method to fakes

* Fix lint

* Fix lint 2

* Use split store method in cleanup

* Add tests

* Remove DeleteExpiredVersions from sqlstore

* Fix lint

* Fix integration tests
This commit is contained in:
idafurjes
2022-05-31 11:56:05 +02:00
committed by GitHub
parent 72367cf1ad
commit f69c9bd704
12 changed files with 177 additions and 172 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/services/sqlstore/dashboard_version.go
-68
View File
@@ -2,10 +2,8 @@ package sqlstore
import (
"context"
"strings"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
)
// GetDashboardVersions gets all dashboard versions for the given dashboard ID.
@@ -42,69 +40,3 @@ func (ss *SQLStore) GetDashboardVersions(ctx context.Context, query *models.GetD
return nil
})
}
const MAX_VERSIONS_TO_DELETE_PER_BATCH = 100
const MAX_VERSION_DELETION_BATCHES = 50
func (ss *SQLStore) DeleteExpiredVersions(ctx context.Context, cmd *models.DeleteExpiredVersionsCommand) error {
return ss.deleteExpiredVersions(ctx, cmd, MAX_VERSIONS_TO_DELETE_PER_BATCH, MAX_VERSION_DELETION_BATCHES)
}
func (ss *SQLStore) deleteExpiredVersions(ctx context.Context, cmd *models.DeleteExpiredVersionsCommand, perBatch int, maxBatches int) error {
versionsToKeep := setting.DashboardVersionsToKeep
if versionsToKeep < 1 {
versionsToKeep = 1
}
for batch := 0; batch < maxBatches; batch++ {
deleted := int64(0)
batchErr := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error {
// Idea of this query is finding version IDs to delete based on formula:
// min_version_to_keep = min_version + (versions_count - versions_to_keep)
// where version stats is processed for each dashboard. This guarantees that we keep at least versions_to_keep
// versions, but in some cases (when versions are sparse) this number may be more.
versionIdsToDeleteQuery := `SELECT id
FROM dashboard_version, (
SELECT dashboard_id, count(version) as count, min(version) as min
FROM dashboard_version
GROUP BY dashboard_id
) AS vtd
WHERE dashboard_version.dashboard_id=vtd.dashboard_id
AND version < vtd.min + vtd.count - ?
LIMIT ?`
var versionIdsToDelete []interface{}
err := sess.SQL(versionIdsToDeleteQuery, versionsToKeep, perBatch).Find(&versionIdsToDelete)
if err != nil {
return err
}
if len(versionIdsToDelete) < 1 {
return nil
}
deleteExpiredSQL := `DELETE FROM dashboard_version WHERE id IN (?` + strings.Repeat(",?", len(versionIdsToDelete)-1) + `)`
sqlOrArgs := append([]interface{}{deleteExpiredSQL}, versionIdsToDelete...)
expiredResponse, err := sess.Exec(sqlOrArgs...)
if err != nil {
return err
}
deleted, err = expiredResponse.RowsAffected()
return err
})
if batchErr != nil {
return batchErr
}
cmd.DeletedRows += deleted
if deleted < int64(perBatch) {
break
}
}
return nil
}
pkg/services/sqlstore/dashboard_version_test.go
-73
View File
@@ -12,7 +12,6 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
)
@@ -110,78 +109,6 @@ func TestIntegrationGetDashboardVersions(t *testing.T) {
})
}
func TestIntegrationDeleteExpiredVersions(t *testing.T) {
versionsToKeep := 5
versionsToWrite := 10
setting.DashboardVersionsToKeep = versionsToKeep
var sqlStore *SQLStore
var savedDash *models.Dashboard
setup := func(t *testing.T) {
sqlStore = InitTestDB(t)
savedDash = insertTestDashboard(t, sqlStore, "test dash 53", 1, 0, false, "diff-all")
for i := 0; i < versionsToWrite-1; i++ {
updateTestDashboard(t, sqlStore, savedDash, map[string]interface{}{
"tags": "different-tag",
})
}
}
t.Run("Clean up old dashboard versions", func(t *testing.T) {
setup(t)
err := sqlStore.DeleteExpiredVersions(context.Background(), &models.DeleteExpiredVersionsCommand{})
require.Nil(t, err)
query := models.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1}
err = sqlStore.GetDashboardVersions(context.Background(), &query)
require.Nil(t, err)
require.Equal(t, versionsToKeep, len(query.Result))
// Ensure latest versions were kept
require.Equal(t, versionsToWrite-versionsToKeep+1, query.Result[versionsToKeep-1].Version)
require.Equal(t, versionsToWrite, query.Result[0].Version)
})
t.Run("Don't delete anything if there are no expired versions", func(t *testing.T) {
setup(t)
setting.DashboardVersionsToKeep = versionsToWrite
err := sqlStore.DeleteExpiredVersions(context.Background(), &models.DeleteExpiredVersionsCommand{})
require.Nil(t, err)
query := models.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1, Limit: versionsToWrite}
err = sqlStore.GetDashboardVersions(context.Background(), &query)
require.Nil(t, err)
require.Equal(t, versionsToWrite, len(query.Result))
})
t.Run("Don't delete more than MAX_VERSIONS_TO_DELETE_PER_BATCH * MAX_VERSION_DELETION_BATCHES per iteration", func(t *testing.T) {
setup(t)
perBatch := 10
maxBatches := 10
versionsToWriteBigNumber := perBatch*maxBatches + versionsToWrite
for i := 0; i < versionsToWriteBigNumber-versionsToWrite; i++ {
updateTestDashboard(t, sqlStore, savedDash, map[string]interface{}{
"tags": "different-tag",
})
}
err := sqlStore.deleteExpiredVersions(context.Background(), &models.DeleteExpiredVersionsCommand{}, perBatch, maxBatches)
require.Nil(t, err)
query := models.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1, Limit: versionsToWriteBigNumber}
err = sqlStore.GetDashboardVersions(context.Background(), &query)
require.Nil(t, err)
// Ensure we have at least versionsToKeep versions
require.GreaterOrEqual(t, len(query.Result), versionsToKeep)
// Ensure we haven't deleted more than perBatch * maxBatches rows
require.LessOrEqual(t, versionsToWriteBigNumber-len(query.Result), perBatch*maxBatches)
})
}
func getDashboard(t *testing.T, sqlStore *SQLStore, dashboard *models.Dashboard) error {
t.Helper()
return sqlStore.WithDbSession(context.Background(), func(sess *DBSession) error {
pkg/services/sqlstore/mockstore/mockstore.go
-4
View File
@@ -348,10 +348,6 @@ func (m *SQLStoreMock) GetDashboardVersions(ctx context.Context, query *models.G
return m.ExpectedError
}
func (m *SQLStoreMock) DeleteExpiredVersions(ctx context.Context, cmd *models.DeleteExpiredVersionsCommand) error {
return m.ExpectedError
}
func (m SQLStoreMock) GetDashboardAclInfoList(ctx context.Context, query *models.GetDashboardAclInfoListQuery) error {
query.Result = m.ExpectedDashboardAclInfoList
return m.ExpectedError
pkg/services/sqlstore/store.go
-1
View File
@@ -75,7 +75,6 @@ type Store interface {
WithTransactionalDbSession(ctx context.Context, callback DBTransactionFunc) error
InTransaction(ctx context.Context, fn func(ctx context.Context) error) error
GetDashboardVersions(ctx context.Context, query *models.GetDashboardVersionsQuery) error
DeleteExpiredVersions(ctx context.Context, cmd *models.DeleteExpiredVersionsCommand) error
GetDashboardAclInfoList(ctx context.Context, query *models.GetDashboardAclInfoListQuery) error
CreatePlaylist(ctx context.Context, cmd *models.CreatePlaylistCommand) error
UpdatePlaylist(ctx context.Context, cmd *models.UpdatePlaylistCommand) error