From 20747015f663662938b9366c65feb1c83ea921f8 Mon Sep 17 00:00:00 2001 From: Carl Bergquist Date: Wed, 2 Sep 2020 08:07:31 +0200 Subject: [PATCH] Annotation: Add clean up job for old annotations (#26156) Co-authored-by: Marcus Efraimsson Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> Co-authored-by: Arve Knudsen --- conf/defaults.ini | 30 +++ conf/sample.ini | 30 +++ docs/sources/administration/configuration.md | 37 ++++ pkg/services/annotations/annotations.go | 25 ++- pkg/services/cleanup/cleanup.go | 14 ++ pkg/services/sqlstore/annotation_cleanup.go | 87 ++++++++ .../sqlstore/annotation_cleanup_test.go | 194 ++++++++++++++++++ pkg/services/sqlstore/sqlstore.go | 1 + pkg/services/sqlstore/sqlutil/sqlutil.go | 1 + pkg/setting/setting.go | 34 +++ 10 files changed, 452 insertions(+), 1 deletion(-) create mode 100644 pkg/services/sqlstore/annotation_cleanup.go create mode 100644 pkg/services/sqlstore/annotation_cleanup_test.go diff --git a/conf/defaults.ini b/conf/defaults.ini index f0c2ccf52f7..44fbbf67930 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -599,6 +599,36 @@ max_attempts = 3 # Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend min_interval_seconds = 1 +# Configures for how long alert annotations are stored. Default is 0, which keeps them forever. +# This setting should be expressed as an duration. Ex 6h (hours), 10d (days), 2w (weeks), 1M (month). +max_annotation_age = + +# Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations. +max_annotations_to_keep = + +#################################### Annotations ######################### + +[annotations.dashboard] +# Dashboard annotations means that annotations are associated with the dashboard they are created on. + +# Configures how long dashboard annotations are stored. Default is 0, which keeps them forever. +# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month). +max_age = + +# Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations. +max_annotations_to_keep = + +[annotations.api] +# API annotations means that the annotations have been created using the API without any +# association with a dashboard. + +# Configures how long Grafana stores API annotations. Default is 0, which keeps them forever. +# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month). +max_age = + +# Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations. +max_annotations_to_keep = + #################################### Explore ############################# [explore] # Enable the Explore section diff --git a/conf/sample.ini b/conf/sample.ini index 9fecf0b06cc..4e415cc05db 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -591,6 +591,36 @@ # Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend ;min_interval_seconds = 1 +# Configures for how long alert annotations are stored. Default is 0, which keeps them forever. +# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month). +;max_annotation_age = + +# Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations. +;max_annotations_to_keep = + +#################################### Annotations ######################### + +[annotations.dashboard] +# Dashboard annotations means that annotations are associated with the dashboard they are created on. + +# Configures how long dashboard annotations are stored. Default is 0, which keeps them forever. +# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month). +;max_age = + +# Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations. +;max_annotations_to_keep = + +[annotations.api] +# API annotations means that the annotations have been created using the API without any +# association with a dashboard. + +# Configures how long Grafana stores API annotations. Default is 0, which keeps them forever. +# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month). +;max_age = + +# Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations. +;max_annotations_to_keep = + #################################### Explore ############################# [explore] # Enable the Explore section diff --git a/docs/sources/administration/configuration.md b/docs/sources/administration/configuration.md index 3fa12227cf9..aa0a0ea2e39 100644 --- a/docs/sources/administration/configuration.md +++ b/docs/sources/administration/configuration.md @@ -976,6 +976,43 @@ Sets the minimum interval between rule evaluations. Default value is `1`. > **Note.** This setting has precedence over each individual rule frequency. If a rule frequency is lower than this value, then this value is enforced. +### max_annotation_age = + +Configures for how long alert annotations are stored. Default is 0, which keeps them forever. +This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month). + +### max_annotations_to_keep = + +Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations. + +
+ +## [annotations.dashboard] + +Dashboard annotations means that annotations are associated with the dashboard they are created on. + +### max_age + +Configures how long dashboard annotations are stored. Default is 0, which keeps them forever. +This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month). + +### max_annotations_to_keep + +Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations. + +## [annotations.api] + +API annotations means that the annotations have been created using the API without any association with a dashboard. + +### max_age + +Configures how long Grafana stores API annotations. Default is 0, which keeps them forever. +This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month). + +### max_annotations_to_keep + +Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations. +
## [explore] diff --git a/pkg/services/annotations/annotations.go b/pkg/services/annotations/annotations.go index 73706ea5850..ceb19d5f5cc 100644 --- a/pkg/services/annotations/annotations.go +++ b/pkg/services/annotations/annotations.go @@ -1,6 +1,11 @@ package annotations -import "github.com/grafana/grafana/pkg/components/simplejson" +import ( + "context" + + "github.com/grafana/grafana/pkg/components/simplejson" + "github.com/grafana/grafana/pkg/setting" +) type Repository interface { Save(item *Item) error @@ -9,6 +14,11 @@ type Repository interface { Delete(params *DeleteParams) error } +// AnnotationCleaner is responsible for cleaning up old annotations +type AnnotationCleaner interface { + CleanAnnotations(ctx context.Context, cfg *setting.Cfg) error +} + type ItemQuery struct { OrgId int64 `json:"orgId"` From int64 `json:"from"` @@ -43,6 +53,15 @@ type DeleteParams struct { } var repositoryInstance Repository +var cleanerInstance AnnotationCleaner + +func GetAnnotationCleaner() AnnotationCleaner { + return cleanerInstance +} + +func SetAnnotationCleaner(rep AnnotationCleaner) { + cleanerInstance = rep +} func GetRepository() Repository { return repositoryInstance @@ -74,6 +93,10 @@ type Item struct { Title string } +func (i Item) TableName() string { + return "annotation" +} + type ItemDTO struct { Id int64 `json:"id"` AlertId int64 `json:"alertId"` diff --git a/pkg/services/cleanup/cleanup.go b/pkg/services/cleanup/cleanup.go index 3b61d0d9c51..f2eb19b6179 100644 --- a/pkg/services/cleanup/cleanup.go +++ b/pkg/services/cleanup/cleanup.go @@ -12,6 +12,7 @@ import ( "github.com/grafana/grafana/pkg/infra/serverlock" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" + "github.com/grafana/grafana/pkg/services/annotations" "github.com/grafana/grafana/pkg/setting" ) @@ -37,9 +38,14 @@ func (srv *CleanUpService) Run(ctx context.Context) error { for { select { case <-ticker.C: + ctxWithTimeout, cancelFn := context.WithTimeout(ctx, time.Minute*9) + defer cancelFn() + srv.cleanUpTmpFiles() srv.deleteExpiredSnapshots() srv.deleteExpiredDashboardVersions() + srv.cleanUpOldAnnotations(ctxWithTimeout) + err := srv.ServerLockService.LockAndExecute(ctx, "delete old login attempts", time.Minute*10, func() { srv.deleteOldLoginAttempts() @@ -53,6 +59,14 @@ func (srv *CleanUpService) Run(ctx context.Context) error { } } +func (srv *CleanUpService) cleanUpOldAnnotations(ctx context.Context) { + cleaner := annotations.GetAnnotationCleaner() + err := cleaner.CleanAnnotations(ctx, srv.Cfg) + if err != nil { + srv.log.Error("failed to clean up old annotations", "error", err) + } +} + func (srv *CleanUpService) cleanUpTmpFiles() { if _, err := os.Stat(srv.Cfg.ImagesDir); os.IsNotExist(err) { return diff --git a/pkg/services/sqlstore/annotation_cleanup.go b/pkg/services/sqlstore/annotation_cleanup.go new file mode 100644 index 00000000000..ad1ba15ece5 --- /dev/null +++ b/pkg/services/sqlstore/annotation_cleanup.go @@ -0,0 +1,87 @@ +package sqlstore + +import ( + "context" + "fmt" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/setting" +) + +// AnnotationCleanupService is responseible for cleaning old annotations. +type AnnotationCleanupService struct { + batchSize int64 + log log.Logger +} + +const ( + alertAnnotationType = "alert_id <> 0" + dashboardAnnotationType = "dashboard_id <> 0 AND alert_id = 0" + apiAnnotationType = "alert_id = 0 AND dashboard_id = 0" +) + +// CleanAnnotations deletes old annotations created by +// alert rules, API requests and human made in the UI. +func (acs *AnnotationCleanupService) CleanAnnotations(ctx context.Context, cfg *setting.Cfg) error { + err := acs.cleanAnnotations(ctx, cfg.AlertingAnnotationCleanupSetting, alertAnnotationType) + if err != nil { + return err + } + + err = acs.cleanAnnotations(ctx, cfg.APIAnnotationCleanupSettings, apiAnnotationType) + if err != nil { + return err + } + + return acs.cleanAnnotations(ctx, cfg.DashboardAnnotationCleanupSettings, dashboardAnnotationType) +} + +func (acs *AnnotationCleanupService) cleanAnnotations(ctx context.Context, cfg setting.AnnotationCleanupSettings, annotationType string) error { + if cfg.MaxAge > 0 { + cutoffDate := time.Now().Add(-cfg.MaxAge).UnixNano() / int64(time.Millisecond) + deleteQuery := `DELETE FROM annotation WHERE id IN (SELECT id FROM (SELECT id FROM annotation WHERE %s AND created < %v ORDER BY id DESC %s) a)` + sql := fmt.Sprintf(deleteQuery, annotationType, cutoffDate, dialect.Limit(acs.batchSize)) + + err := acs.executeUntilDoneOrCancelled(ctx, sql) + if err != nil { + return err + } + } + + if cfg.MaxCount > 0 { + deleteQuery := `DELETE FROM annotation WHERE id IN (SELECT id FROM (SELECT id FROM annotation WHERE %s ORDER BY id DESC %s) a)` + sql := fmt.Sprintf(deleteQuery, annotationType, dialect.LimitOffset(acs.batchSize, cfg.MaxCount)) + return acs.executeUntilDoneOrCancelled(ctx, sql) + } + + return nil +} + +func (acs *AnnotationCleanupService) executeUntilDoneOrCancelled(ctx context.Context, sql string) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + var affected int64 + err := withDbSession(ctx, func(session *DBSession) error { + res, err := session.Exec(sql) + if err != nil { + return err + } + + affected, err = res.RowsAffected() + + return err + }) + if err != nil { + return err + } + + if affected == 0 { + return nil + } + } + } +} diff --git a/pkg/services/sqlstore/annotation_cleanup_test.go b/pkg/services/sqlstore/annotation_cleanup_test.go new file mode 100644 index 00000000000..16fe7053f1f --- /dev/null +++ b/pkg/services/sqlstore/annotation_cleanup_test.go @@ -0,0 +1,194 @@ +package sqlstore + +import ( + "context" + "testing" + "time" + + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/annotations" + "github.com/grafana/grafana/pkg/setting" + "github.com/stretchr/testify/require" +) + +func TestAnnotationCleanUp(t *testing.T) { + fakeSQL := InitTestDB(t) + + t.Cleanup(func() { + _ = fakeSQL.WithDbSession(context.Background(), func(session *DBSession) error { + _, err := session.Exec("DELETE FROM annotation") + require.Nil(t, err, "cleaning up all annotations should not cause problems") + return err + }) + }) + + createTestAnnotations(t, fakeSQL, 21, 6) + assertAnnotationCount(t, fakeSQL, "", 21) + + tests := []struct { + name string + cfg *setting.Cfg + alertAnnotationCount int64 + dashboardAnnotationCount int64 + APIAnnotationCount int64 + }{ + { + name: "default settings should not delete any annotations", + cfg: &setting.Cfg{ + AlertingAnnotationCleanupSetting: settingsFn(0, 0), + DashboardAnnotationCleanupSettings: settingsFn(0, 0), + APIAnnotationCleanupSettings: settingsFn(0, 0), + }, + alertAnnotationCount: 7, + dashboardAnnotationCount: 7, + APIAnnotationCount: 7, + }, + { + name: "should remove annotations created before cut off point", + cfg: &setting.Cfg{ + AlertingAnnotationCleanupSetting: settingsFn(time.Hour*48, 0), + DashboardAnnotationCleanupSettings: settingsFn(time.Hour*48, 0), + APIAnnotationCleanupSettings: settingsFn(time.Hour*48, 0), + }, + alertAnnotationCount: 5, + dashboardAnnotationCount: 5, + APIAnnotationCount: 5, + }, + { + name: "should only keep three annotations", + cfg: &setting.Cfg{ + AlertingAnnotationCleanupSetting: settingsFn(0, 3), + DashboardAnnotationCleanupSettings: settingsFn(0, 3), + APIAnnotationCleanupSettings: settingsFn(0, 3), + }, + alertAnnotationCount: 3, + dashboardAnnotationCount: 3, + APIAnnotationCount: 3, + }, + { + name: "running the max count delete again should not remove any annotations", + cfg: &setting.Cfg{ + AlertingAnnotationCleanupSetting: settingsFn(0, 3), + DashboardAnnotationCleanupSettings: settingsFn(0, 3), + APIAnnotationCleanupSettings: settingsFn(0, 3), + }, + alertAnnotationCount: 3, + dashboardAnnotationCount: 3, + APIAnnotationCount: 3, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger")} + err := cleaner.CleanAnnotations(context.Background(), test.cfg) + require.NoError(t, err) + + assertAnnotationCount(t, fakeSQL, alertAnnotationType, test.alertAnnotationCount) + assertAnnotationCount(t, fakeSQL, dashboardAnnotationType, test.dashboardAnnotationCount) + assertAnnotationCount(t, fakeSQL, apiAnnotationType, test.APIAnnotationCount) + }) + } +} + +func TestOldAnnotationsAreDeletedFirst(t *testing.T) { + fakeSQL := InitTestDB(t) + + t.Cleanup(func() { + _ = fakeSQL.WithDbSession(context.Background(), func(session *DBSession) error { + _, err := session.Exec("DELETE FROM annotation") + require.Nil(t, err, "cleaning up all annotations should not cause problems") + return err + }) + }) + + //create some test annotations + a := annotations.Item{ + DashboardId: 1, + OrgId: 1, + UserId: 1, + PanelId: 1, + AlertId: 10, + Text: "", + Created: time.Now().AddDate(-10, 0, -10).UnixNano() / int64(time.Millisecond), + } + + session := fakeSQL.NewSession() + defer session.Close() + + _, err := session.Insert(a) + require.NoError(t, err, "cannot insert annotation") + _, err = session.Insert(a) + require.NoError(t, err, "cannot insert annotation") + + a.AlertId = 20 + _, err = session.Insert(a) + require.NoError(t, err, "cannot insert annotation") + + // run the clean up task to keep one annotation. + cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger")} + err = cleaner.cleanAnnotations(context.Background(), setting.AnnotationCleanupSettings{MaxCount: 1}, alertAnnotationType) + require.NoError(t, err) + + // assert that the last annotations were kept + countNew, err := session.Where("alert_id = 20").Count(&annotations.Item{}) + require.NoError(t, err) + require.Equal(t, int64(1), countNew, "the last annotations should be kept") + + countOld, err := session.Where("alert_id = 10").Count(&annotations.Item{}) + require.NoError(t, err) + require.Equal(t, int64(0), countOld, "the two first annotations should have been deleted.") +} + +func assertAnnotationCount(t *testing.T, fakeSQL *SqlStore, sql string, expectedCount int64) { + t.Helper() + + session := fakeSQL.NewSession() + defer session.Close() + count, err := session.Where(sql).Count(&annotations.Item{}) + require.NoError(t, err) + require.Equal(t, expectedCount, count) +} + +func createTestAnnotations(t *testing.T, sqlstore *SqlStore, expectedCount int, oldAnnotations int) { + t.Helper() + + cutoffDate := time.Now() + + for i := 0; i < expectedCount; i++ { + a := &annotations.Item{ + DashboardId: 1, + OrgId: 1, + UserId: 1, + PanelId: 1, + Text: "", + } + + // mark every third as an API annotation + // that doesnt belong to a dashboard + if i%3 == 1 { + a.DashboardId = 0 + } + + // mark every third annotation as an alert annotation + if i%3 == 0 { + a.AlertId = 10 + a.DashboardId = 2 + } + + // create epoch as int annotations.go line 40 + a.Created = cutoffDate.UnixNano() / int64(time.Millisecond) + + // set a really old date for the first six annotations + if i < oldAnnotations { + a.Created = cutoffDate.AddDate(-10, 0, -10).UnixNano() / int64(time.Millisecond) + } + + _, err := sqlstore.NewSession().Insert(a) + require.NoError(t, err, "should be able to save annotation", err) + } +} + +func settingsFn(maxAge time.Duration, maxCount int64) setting.AnnotationCleanupSettings { + return setting.AnnotationCleanupSettings{MaxAge: maxAge, MaxCount: maxCount} +} diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go index 59efd4d6680..406d2e194db 100644 --- a/pkg/services/sqlstore/sqlstore.go +++ b/pkg/services/sqlstore/sqlstore.go @@ -96,6 +96,7 @@ func (ss *SqlStore) Init() error { // Init repo instances annotations.SetRepository(&SqlAnnotationRepo{}) + annotations.SetAnnotationCleaner(&AnnotationCleanupService{batchSize: 100, log: log.New("annotationcleaner")}) ss.Bus.SetTransactionManager(ss) // Register handlers diff --git a/pkg/services/sqlstore/sqlutil/sqlutil.go b/pkg/services/sqlstore/sqlutil/sqlutil.go index 7416bc801dd..98363bf1cba 100644 --- a/pkg/services/sqlstore/sqlutil/sqlutil.go +++ b/pkg/services/sqlstore/sqlutil/sqlutil.go @@ -11,6 +11,7 @@ type TestDB struct { } func Sqlite3TestDB() TestDB { + // To run all tests in a local test database, set ConnStr to "grafana_test.db" return TestDB{ DriverName: "sqlite3", ConnStr: ":memory:", diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index fe88656c6c7..dfc3f5d7744 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -19,6 +19,7 @@ import ( "github.com/go-macaron/session" ini "gopkg.in/ini.v1" + "github.com/grafana/grafana/pkg/components/gtime" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/util" ) @@ -302,6 +303,11 @@ type Cfg struct { FeatureToggles map[string]bool AnonymousHideVersion bool + + // Annotations + AlertingAnnotationCleanupSetting AnnotationCleanupSettings + DashboardAnnotationCleanupSettings AnnotationCleanupSettings + APIAnnotationCleanupSettings AnnotationCleanupSettings } // IsExpressionsEnabled returns whether the expressions feature is enabled. @@ -396,6 +402,33 @@ func applyEnvVariableOverrides(file *ini.File) error { return nil } +func (cfg *Cfg) readAnnotationSettings() { + dashboardAnnotation := cfg.Raw.Section("annotations.dashboard") + apiIAnnotation := cfg.Raw.Section("annotations.api") + alertingSection := cfg.Raw.Section("alerting") + + var newAnnotationCleanupSettings = func(section *ini.Section, maxAgeField string) AnnotationCleanupSettings { + maxAge, err := gtime.ParseInterval(section.Key(maxAgeField).MustString("")) + if err != nil { + maxAge = 0 + } + + return AnnotationCleanupSettings{ + MaxAge: maxAge, + MaxCount: section.Key("max_annotations_to_keep").MustInt64(0), + } + } + + cfg.AlertingAnnotationCleanupSetting = newAnnotationCleanupSettings(alertingSection, "max_annotation_age") + cfg.DashboardAnnotationCleanupSettings = newAnnotationCleanupSettings(dashboardAnnotation, "max_age") + cfg.APIAnnotationCleanupSettings = newAnnotationCleanupSettings(apiIAnnotation, "max_age") +} + +type AnnotationCleanupSettings struct { + MaxAge time.Duration + MaxCount int64 +} + func envKey(sectionName string, keyName string) string { sN := strings.ToUpper(strings.Replace(sectionName, ".", "_", -1)) sN = strings.Replace(sN, "-", "_", -1) @@ -758,6 +791,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error { cfg.readSessionConfig() cfg.readSmtpSettings() cfg.readQuotaSettings() + cfg.readAnnotationSettings() if VerifyEmailEnabled && !cfg.Smtp.Enabled { log.Warnf("require_email_validation is enabled but smtp is disabled")