snapshots: fixes cleanup of old snapshots

Snapshot cleanup did not work due to time.Now syntax error. Added test
for it as well to catch any future errors.

Added error and debug logging so that it is possible to see any errors in the future.

Removed an unused configuration value and deprecated the remove expired snapshots
setting.
This commit is contained in:
Daniel Lee 2018-02-20 23:10:59 +01:00
parent 7d9a9fa29c
commit fe49182b9d
10 changed files with 94 additions and 27 deletions

View File

@ -187,9 +187,6 @@ external_snapshot_name = Publish to snapshot.raintank.io
# remove expired snapshot
snapshot_remove_expired = true
# remove snapshots after 90 days
snapshot_TTL_days = 90
#################################### Dashboards ##################
[dashboards]

View File

@ -175,9 +175,6 @@ log_queries =
# remove expired snapshot
;snapshot_remove_expired = true
# remove snapshots after 90 days
;snapshot_TTL_days = 90
#################################### Dashboards History ##################
[dashboards]
# Number dashboard versions to keep (per dashboard). Default: 20, Minimum: 1

View File

@ -795,12 +795,9 @@ Set root url to a Grafana instance where you want to publish external snapshots
### external_snapshot_name
Set name for external snapshot button. Defaults to `Publish to snapshot.raintank.io`
### remove expired snapshot
### snapshot_remove_expired
Enabled to automatically remove expired snapshots
### remove snapshots after 90 days
Time to live for snapshots.
## [external_image_storage]
These options control how images should be made public so they can be shared on services like slack.

View File

@ -64,6 +64,7 @@ type DeleteDashboardSnapshotCommand struct {
}
type DeleteExpiredSnapshotsCommand struct {
DeletedRows int64
}
type GetDashboardSnapshotQuery struct {

View File

@ -75,4 +75,5 @@ type GetDashboardVersionsQuery struct {
//
type DeleteExpiredVersionsCommand struct {
DeletedRows int64
}

View File

@ -83,11 +83,21 @@ func (service *CleanUpService) cleanUpTmpFiles() {
}
func (service *CleanUpService) deleteExpiredSnapshots() {
bus.Dispatch(&m.DeleteExpiredSnapshotsCommand{})
cmd := m.DeleteExpiredSnapshotsCommand{}
if err := bus.Dispatch(&cmd); err != nil {
service.log.Error("Failed to delete expired snapshots", "error", err.Error())
} else {
service.log.Debug("Deleted expired snapshots", "rows affected", cmd.DeletedRows)
}
}
func (service *CleanUpService) deleteExpiredDashboardVersions() {
bus.Dispatch(&m.DeleteExpiredVersionsCommand{})
cmd := m.DeleteExpiredVersionsCommand{}
if err := bus.Dispatch(&cmd); err != nil {
service.log.Error("Failed to delete expired dashboard versions", "error", err.Error())
} else {
service.log.Debug("Deleted old/expired dashboard versions", "rows affected", cmd.DeletedRows)
}
}
func (service *CleanUpService) deleteOldLoginAttempts() {

View File

@ -16,20 +16,23 @@ func init() {
bus.AddHandler("sql", DeleteExpiredSnapshots)
}
// DeleteExpiredSnapshots removes snapshots with old expiry dates.
// SnapShotRemoveExpired is deprecated and should be removed in the future.
// Snapshot expiry is decided by the user when they share the snapshot.
func DeleteExpiredSnapshots(cmd *m.DeleteExpiredSnapshotsCommand) error {
return inTransaction(func(sess *DBSession) error {
var expiredCount int64 = 0
if setting.SnapShotRemoveExpired {
deleteExpiredSql := "DELETE FROM dashboard_snapshot WHERE expires < ?"
expiredResponse, err := x.Exec(deleteExpiredSql, time.Now)
if err != nil {
return err
}
expiredCount, _ = expiredResponse.RowsAffected()
if !setting.SnapShotRemoveExpired {
sqlog.Warn("[Deprecated] The snapshot_remove_expired setting is outdated. Please remove from your config.")
return nil
}
sqlog.Debug("Deleted old/expired snaphots", "expired", expiredCount)
deleteExpiredSql := "DELETE FROM dashboard_snapshot WHERE expires < ?"
expiredResponse, err := sess.Exec(deleteExpiredSql, time.Now())
if err != nil {
return err
}
cmd.DeletedRows, _ = expiredResponse.RowsAffected()
return nil
})
}

View File

@ -2,11 +2,14 @@ package sqlstore
import (
"testing"
"time"
"github.com/go-xorm/xorm"
. "github.com/smartystreets/goconvey/convey"
"github.com/grafana/grafana/pkg/components/simplejson"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
)
func TestDashboardSnapshotDBAccess(t *testing.T) {
@ -105,3 +108,65 @@ func TestDashboardSnapshotDBAccess(t *testing.T) {
})
})
}
func TestDeleteExpiredSnapshots(t *testing.T) {
Convey("Testing dashboard snapshots clean up", t, func() {
x := InitTestDB(t)
setting.SnapShotRemoveExpired = true
notExpiredsnapshot := createTestSnapshot(x, "key1", 1000)
createTestSnapshot(x, "key2", -1000)
createTestSnapshot(x, "key3", -1000)
Convey("Clean up old dashboard snapshots", func() {
err := DeleteExpiredSnapshots(&m.DeleteExpiredSnapshotsCommand{})
So(err, ShouldBeNil)
query := m.GetDashboardSnapshotsQuery{
OrgId: 1,
SignedInUser: &m.SignedInUser{OrgRole: m.ROLE_ADMIN},
}
err = SearchDashboardSnapshots(&query)
So(err, ShouldBeNil)
So(len(query.Result), ShouldEqual, 1)
So(query.Result[0].Key, ShouldEqual, notExpiredsnapshot.Key)
})
Convey("Don't delete anything if there are no expired snapshots", func() {
err := DeleteExpiredSnapshots(&m.DeleteExpiredSnapshotsCommand{})
So(err, ShouldBeNil)
query := m.GetDashboardSnapshotsQuery{
OrgId: 1,
SignedInUser: &m.SignedInUser{OrgRole: m.ROLE_ADMIN},
}
SearchDashboardSnapshots(&query)
So(len(query.Result), ShouldEqual, 1)
})
})
}
func createTestSnapshot(x *xorm.Engine, key string, expires int64) *m.DashboardSnapshot {
cmd := m.CreateDashboardSnapshotCommand{
Key: key,
DeleteKey: "delete" + key,
Dashboard: simplejson.NewFromAny(map[string]interface{}{
"hello": "mupp",
}),
UserId: 1000,
OrgId: 1,
Expires: expires,
}
err := CreateDashboardSnapshot(&cmd)
So(err, ShouldBeNil)
// Set expiry date manually - to be able to create expired snapshots
expireDate := time.Now().Add(time.Second * time.Duration(expires))
_, err = x.Exec("update dashboard_snapshot set expires = ? where "+dialect.Quote("key")+" = ?", expireDate, key)
So(err, ShouldBeNil)
return cmd.Result
}

View File

@ -69,7 +69,6 @@ func GetDashboardVersions(query *m.GetDashboardVersionsQuery) error {
func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error {
return inTransaction(func(sess *DBSession) error {
expiredCount := int64(0)
versions := []DashboardVersionExp{}
versionsToKeep := setting.DashboardVersionsToKeep
@ -98,8 +97,7 @@ func DeleteExpiredVersions(cmd *m.DeleteExpiredVersionsCommand) error {
if err != nil {
return err
}
expiredCount, _ = expiredResponse.RowsAffected()
sqlog.Debug("Deleted old/expired dashboard versions", "expired", expiredCount)
cmd.DeletedRows, _ = expiredResponse.RowsAffected()
}
return nil

View File

@ -88,7 +88,6 @@ var (
ExternalSnapshotUrl string
ExternalSnapshotName string
ExternalEnabled bool
SnapShotTTLDays int
SnapShotRemoveExpired bool
// Dashboard history
@ -523,7 +522,6 @@ func NewConfigContext(args *CommandLineArgs) error {
ExternalSnapshotName = snapshots.Key("external_snapshot_name").String()
ExternalEnabled = snapshots.Key("external_enabled").MustBool(true)
SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true)
SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90)
// read dashboard settings
dashboards := Cfg.Section("dashboards")