diff --git a/pkg/services/ngalert/schedule/compat.go b/pkg/services/ngalert/schedule/compat.go index e78877ea036..a771987b59f 100644 --- a/pkg/services/ngalert/schedule/compat.go +++ b/pkg/services/ngalert/schedule/compat.go @@ -28,34 +28,35 @@ func FromAlertStateToPostableAlerts(logger log.Logger, firingStates []*state.Sta } for _, alertState := range firingStates { - if alertState.NeedsSending(stateManager.ResendDelay) { - nL := alertState.Labels.Copy() - nA := data.Labels(alertState.Annotations).Copy() - - if len(alertState.Results) > 0 { - nA["__value_string__"] = alertState.Results[0].EvaluationString - } - - genURL := appURL - if uid := nL[ngModels.RuleUIDLabel]; len(uid) > 0 && u != nil { - oldPath := u.Path - u.Path = path.Join(u.Path, fmt.Sprintf("/alerting/%s/edit", uid)) - genURL = u.String() - u.Path = oldPath - } - - alerts.PostableAlerts = append(alerts.PostableAlerts, models.PostableAlert{ - Annotations: models.LabelSet(nA), - StartsAt: strfmt.DateTime(alertState.StartsAt), - EndsAt: strfmt.DateTime(alertState.EndsAt), - Alert: models.Alert{ - Labels: models.LabelSet(nL), - GeneratorURL: strfmt.URI(genURL), - }, - }) - alertState.LastSentAt = ts - sentAlerts = append(sentAlerts, alertState) + if !alertState.NeedsSending(stateManager.ResendDelay) { + continue } + nL := alertState.Labels.Copy() + nA := data.Labels(alertState.Annotations).Copy() + + if len(alertState.Results) > 0 { + nA["__value_string__"] = alertState.Results[0].EvaluationString + } + + genURL := appURL + if uid := nL[ngModels.RuleUIDLabel]; len(uid) > 0 && u != nil { + oldPath := u.Path + u.Path = path.Join(u.Path, fmt.Sprintf("/alerting/%s/edit", uid)) + genURL = u.String() + u.Path = oldPath + } + + alerts.PostableAlerts = append(alerts.PostableAlerts, models.PostableAlert{ + Annotations: models.LabelSet(nA), + StartsAt: strfmt.DateTime(alertState.StartsAt), + EndsAt: strfmt.DateTime(alertState.EndsAt), + Alert: models.Alert{ + Labels: models.LabelSet(nL), + GeneratorURL: strfmt.URI(genURL), + }, + }) + alertState.LastSentAt = ts + sentAlerts = append(sentAlerts, alertState) } stateManager.Put(sentAlerts) return alerts diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 7e1d86725b3..52e4afd4669 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -175,6 +175,10 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul case eval.Pending: // we do not emit results with this state } + // Set Resolved property so the scheduler knows to send a postable alert + // to Alertmanager. + currentState.Resolved = oldState == eval.Alerting && currentState.State == eval.Normal + st.set(currentState) if oldState != currentState.State { go st.createAlertAnnotation(currentState.State, alertRule, result) diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index a622ee85e3a..85f8f6f7b8f 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -13,6 +13,7 @@ type State struct { OrgID int64 CacheId string State eval.State + Resolved bool Results []Evaluation StartsAt time.Time EndsAt time.Time @@ -112,10 +113,13 @@ func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result) } func (a *State) NeedsSending(resendDelay time.Duration) bool { - if a.State != eval.Alerting { + if a.State != eval.Alerting && a.State != eval.Normal { return false } + if a.State == eval.Normal && !a.Resolved { + return false + } // if LastSentAt is before or equal to LastEvaluationTime + resendDelay, send again return a.LastSentAt.Add(resendDelay).Before(a.LastEvaluationTime) || a.LastSentAt.Add(resendDelay).Equal(a.LastEvaluationTime) diff --git a/pkg/services/ngalert/state/state_test.go b/pkg/services/ngalert/state/state_test.go index cf87f062457..89c0f663632 100644 --- a/pkg/services/ngalert/state/state_test.go +++ b/pkg/services/ngalert/state/state_test.go @@ -67,6 +67,39 @@ func TestNeedsSending(t *testing.T) { LastSentAt: evaluationTime, }, }, + { + name: "state: normal + resolved sends after a minute", + resendDelay: 1 * time.Minute, + expected: true, + testState: &State{ + State: eval.Normal, + Resolved: true, + LastEvaluationTime: evaluationTime, + LastSentAt: evaluationTime.Add(-1 * time.Minute), + }, + }, + { + name: "state: normal + resolved does _not_ send after 30 seconds (before one minute)", + resendDelay: 1 * time.Minute, + expected: false, + testState: &State{ + State: eval.Normal, + Resolved: true, + LastEvaluationTime: evaluationTime, + LastSentAt: evaluationTime.Add(-30 * time.Second), + }, + }, + { + name: "state: normal but not resolved does not send after a minute", + resendDelay: 1 * time.Minute, + expected: false, + testState: &State{ + State: eval.Normal, + Resolved: false, + LastEvaluationTime: evaluationTime, + LastSentAt: evaluationTime.Add(-1 * time.Minute), + }, + }, } for _, tc := range testCases {