diff --git a/.gitignore b/.gitignore index fbd92fe0b51..719f4347779 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ profile.cov /pkg/cmd/grafana-cli/grafana-cli /pkg/cmd/grafana-server/grafana-server /pkg/cmd/grafana-server/debug +debug.test /examples/*/dist /packaging/**/*.rpm /packaging/**/*.deb diff --git a/conf/defaults.ini b/conf/defaults.ini index 3602c156c6b..2709e36882c 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -174,6 +174,7 @@ disable_gravatar = false # data source proxy whitelist (ip_or_domain:port separated by spaces) data_source_proxy_whitelist = +#################################### Snapshots ########################### [snapshots] # snapshot sharing options external_enabled = true @@ -186,7 +187,13 @@ snapshot_remove_expired = true # remove snapshots after 90 days snapshot_TTL_days = 90 -#################################### Users #################################### +#################################### Dashboards History ################## +[dashboards] +# A setting of 20 (default) means only the last 20 versions will be stored and older versions removed. +# To keep all dashboard versions you can set this value to 2147483647. +versions_to_keep = 20 + +#################################### Users ############################### [users] # disable user signup / registration allow_sign_up = false diff --git a/conf/sample.ini b/conf/sample.ini index 0787267cb8d..51451039e0a 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -162,6 +162,7 @@ log_queries = # data source proxy whitelist (ip_or_domain:port separated by spaces) ;data_source_proxy_whitelist = +#################################### Snapshots ########################### [snapshots] # snapshot sharing options ;external_enabled = true @@ -174,7 +175,13 @@ log_queries = # remove snapshots after 90 days ;snapshot_TTL_days = 90 -#################################### Users #################################### +#################################### Dashboards History ################## +[dashboards] +# A setting of 20 (default) means only the last 20 versions will be stored and older versions removed. +# To keep all dashboard versions you can set this value to 2147483647. +;versions_to_keep = 20 + +#################################### Users ############################### [users] # disable user signup / registration ;allow_sign_up = true diff --git a/pkg/models/dashboard_version.go b/pkg/models/dashboard_version.go index 06b5797e57c..4acb4282a58 100644 --- a/pkg/models/dashboard_version.go +++ b/pkg/models/dashboard_version.go @@ -69,3 +69,10 @@ type GetDashboardVersionsQuery struct { Result []*DashboardVersionDTO } + +// +// Commands +// + +type DeleteExpiredVersionsCommand struct { +} diff --git a/pkg/services/cleanup/cleanup.go b/pkg/services/cleanup/cleanup.go index ffaa75de9cc..1b85a1819e0 100644 --- a/pkg/services/cleanup/cleanup.go +++ b/pkg/services/cleanup/cleanup.go @@ -45,6 +45,7 @@ func (service *CleanUpService) start(ctx context.Context) error { case <-ticker.C: service.cleanUpTmpFiles() service.deleteExpiredSnapshots() + service.deleteExpiredDashboardVersions() case <-ctx.Done(): return ctx.Err() } @@ -83,3 +84,7 @@ func (service *CleanUpService) cleanUpTmpFiles() { func (service *CleanUpService) deleteExpiredSnapshots() { bus.Dispatch(&m.DeleteExpiredSnapshotsCommand{}) } + +func (service *CleanUpService) deleteExpiredDashboardVersions() { + bus.Dispatch(&m.DeleteExpiredVersionsCommand{}) +} diff --git a/pkg/services/sqlstore/dashboard_version.go b/pkg/services/sqlstore/dashboard_version.go index 0b296650b4b..0b84e23006e 100644 --- a/pkg/services/sqlstore/dashboard_version.go +++ b/pkg/services/sqlstore/dashboard_version.go @@ -1,13 +1,18 @@ package sqlstore import ( + "math" + "strings" + "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" ) func init() { bus.AddHandler("sql", GetDashboardVersion) bus.AddHandler("sql", GetDashboardVersions) + bus.AddHandler("sql", DeleteExpiredVersions) } // GetDashboardVersion gets the dashboard version for the given dashboard ID and version number. @@ -58,3 +63,71 @@ func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error { } return nil } + +func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error { + return inTransaction(func(sess *DBSession) error { + var expiredCount int64 = 0 + var versions []DashboardVersionExp + + // Don't clean up if user set versions_to_keep to 2147483647 (MaxInt32) + if versionsToKeep := setting.DashboardVersionsToKeep; versionsToKeep < math.MaxInt32 { + err := sess.Table("dashboard_version"). + Select("dashboard_version.id, dashboard_version.version, dashboard_version.dashboard_id"). + Where(`dashboard_id IN ( + SELECT dashboard_id FROM dashboard_version + GROUP BY dashboard_id HAVING COUNT(dashboard_version.id) > ? + )`, versionsToKeep). + Desc("dashboard_version.dashboard_id", "dashboard_version.version"). + Find(&versions) + + if err != nil { + return err + } + + // Keep last versionsToKeep versions and delete other + versionIdsToDelete := getVersionIDsToDelete(versions, versionsToKeep) + if len(versionIdsToDelete) > 0 { + deleteExpiredSql := `DELETE FROM dashboard_version WHERE id IN (?` + strings.Repeat(",?", len(versionIdsToDelete)-1) + `)` + expiredResponse, err := sess.Exec(deleteExpiredSql, versionIdsToDelete...) + if err != nil { + return err + } + expiredCount, _ = expiredResponse.RowsAffected() + } + } + + sqlog.Debug("Deleted old/expired dashboard versions", "expired", expiredCount) + return nil + }) +} + +// Short version of DashboardVersion for getting expired versions +type DashboardVersionExp struct { + Id int64 `json:"id"` + DashboardId int64 `json:"dashboardId"` + Version int `json:"version"` +} + +func getVersionIDsToDelete(versions []DashboardVersionExp, versionsToKeep int) []interface{} { + versionIds := make([]interface{}, 0) + + if len(versions) == 0 { + return versionIds + } + + currentDashboard := versions[0].DashboardId + count := 0 + for _, v := range versions { + if v.DashboardId == currentDashboard { + count++ + } else { + count = 1 + currentDashboard = v.DashboardId + } + if count > versionsToKeep { + versionIds = append(versionIds, v.Id) + } + } + + return versionIds +} diff --git a/pkg/services/sqlstore/dashboard_version_test.go b/pkg/services/sqlstore/dashboard_version_test.go index 8ac636c1682..a27d4385637 100644 --- a/pkg/services/sqlstore/dashboard_version_test.go +++ b/pkg/services/sqlstore/dashboard_version_test.go @@ -8,6 +8,7 @@ import ( "github.com/grafana/grafana/pkg/components/simplejson" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" ) func updateTestDashboard(dashboard *m.Dashboard, data map[string]interface{}) { @@ -101,3 +102,44 @@ func TestGetDashboardVersions(t *testing.T) { }) }) } + +func TestDeleteExpiredVersions(t *testing.T) { + Convey("Testing dashboard versions clean up", t, func() { + InitTestDB(t) + versionsToKeep := 5 + versionsToWrite := 10 + setting.DashboardVersionsToKeep = versionsToKeep + + savedDash := insertTestDashboard("test dash 53", 1, "diff-all") + for i := 0; i < versionsToWrite-1; i++ { + updateTestDashboard(savedDash, map[string]interface{}{ + "tags": "different-tag", + }) + } + + Convey("Clean up old dashboard versions", func() { + err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{}) + So(err, ShouldBeNil) + + query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1} + GetDashboardVersions(&query) + + So(len(query.Result), ShouldEqual, versionsToKeep) + // Ensure latest versions were kept + So(query.Result[versionsToKeep-1].Version, ShouldEqual, versionsToWrite-versionsToKeep+1) + So(query.Result[0].Version, ShouldEqual, versionsToWrite) + }) + + Convey("Don't delete anything if there're no expired versions", func() { + setting.DashboardVersionsToKeep = versionsToWrite + + err := DeleteExpiredVersions(&m.DeleteExpiredVersionsCommand{}) + So(err, ShouldBeNil) + + query := m.GetDashboardVersionsQuery{DashboardId: savedDash.Id, OrgId: 1} + GetDashboardVersions(&query) + + So(len(query.Result), ShouldEqual, versionsToWrite) + }) + }) +} diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index 95e88fd7190..2caf7366727 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -90,6 +90,9 @@ var ( SnapShotTTLDays int SnapShotRemoveExpired bool + // Dashboard history + DashboardVersionsToKeep int + // User settings AllowUserSignUp bool AllowUserOrgCreate bool @@ -520,6 +523,10 @@ func NewConfigContext(args *CommandLineArgs) error { SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true) SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90) + // read dashboard settings + dashboards := Cfg.Section("dashboards") + DashboardVersionsToKeep = dashboards.Key("versions_to_keep").MustInt(20) + // read data source proxy white list DataProxyWhiteList = make(map[string]bool) for _, hostAndIp := range util.SplitString(security.Key("data_source_proxy_whitelist").String()) {