diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 0f8e24bcef5..43f6db66771 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -10,7 +10,9 @@ import ( tlog "github.com/opentracing/opentracing-go/log" "github.com/benbjohnson/clock" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" + m "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/registry" "github.com/grafana/grafana/pkg/services/rendering" "github.com/grafana/grafana/pkg/setting" @@ -205,6 +207,18 @@ func (e *AlertingService) processJob(attemptID int, attemptChan chan int, cancel } evalContext.Rule.State = evalContext.GetNewState() + if evalContext.Rule.State == m.AlertStateOK && evalContext.PrevAlertState != m.AlertStateOK { + for _, notifierId := range evalContext.Rule.Notifications { + cmd := &m.CleanNotificationJournalCommand{ + AlertId: evalContext.Rule.Id, + NotifierId: notifierId, + OrgId: evalContext.Rule.OrgId, + } + if err := bus.Dispatch(cmd); err != nil { + e.log.Error("Failed to clean up old notification records", "notifier", notifierId, "alert", evalContext.Rule.Id, "Error", err) + } + } + } e.resultHandler.Handle(evalContext) span.Finish() e.log.Debug("Job Execution completed", "timeMs", evalContext.GetDurationMs(), "alertId", evalContext.Rule.Id, "name", evalContext.Rule.Name, "firing", evalContext.Firing, "attemptID", attemptID) diff --git a/pkg/services/alerting/notifiers/base.go b/pkg/services/alerting/notifiers/base.go index 734b5e56b28..7672d397491 100644 --- a/pkg/services/alerting/notifiers/base.go +++ b/pkg/services/alerting/notifiers/base.go @@ -45,6 +45,10 @@ func defaultShouldNotify(context *alerting.EvalContext, notifyOnce bool, frequen if !notifyOnce && lastNotify != nil && lastNotify.Add(frequency).After(time.Now()) { return false } + // Do not notify if alert state if OK or pending even on repeated notify + if !notifyOnce && (context.Rule.State == m.AlertStateOK || context.Rule.State == m.AlertStatePending) { + return false + } // Do not notify when we become OK for the first time. if (context.PrevAlertState == m.AlertStatePending) && (context.Rule.State == m.AlertStateOK) { return false