NGAlert: Send resolve signal to alertmanager on alerting -> Normal (#37363)

pkg/services/ngalert/schedule/compat.go

@@ -28,7 +28,9 @@ func FromAlertStateToPostableAlerts(logger log.Logger, firingStates []*state.Sta
 	}
 
 	for _, alertState := range firingStates {
-		if alertState.NeedsSending(stateManager.ResendDelay) {
+		if !alertState.NeedsSending(stateManager.ResendDelay) {
+			continue
+		}
 		nL := alertState.Labels.Copy()
 		nA := data.Labels(alertState.Annotations).Copy()
 
@@ -56,7 +58,6 @@ func FromAlertStateToPostableAlerts(logger log.Logger, firingStates []*state.Sta
 		alertState.LastSentAt = ts
 		sentAlerts = append(sentAlerts, alertState)
 	}
-	}
 	stateManager.Put(sentAlerts)
 	return alerts
 }

pkg/services/ngalert/state/manager.go

@@ -175,6 +175,10 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul
 	case eval.Pending: // we do not emit results with this state
 	}
 
+	// Set Resolved property so the scheduler knows to send a postable alert
+	// to Alertmanager.
+	currentState.Resolved = oldState == eval.Alerting && currentState.State == eval.Normal
+
 	st.set(currentState)
 	if oldState != currentState.State {
 		go st.createAlertAnnotation(currentState.State, alertRule, result)

pkg/services/ngalert/state/state.go

@@ -13,6 +13,7 @@ type State struct {
 	OrgID              int64
 	CacheId            string
 	State              eval.State
+	Resolved           bool
 	Results            []Evaluation
 	StartsAt           time.Time
 	EndsAt             time.Time
@@ -112,10 +113,13 @@ func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result)
 }
 
 func (a *State) NeedsSending(resendDelay time.Duration) bool {
-	if a.State != eval.Alerting {
+	if a.State != eval.Alerting && a.State != eval.Normal {
 		return false
 	}
 
+	if a.State == eval.Normal && !a.Resolved {
+		return false
+	}
 	// if LastSentAt is before or equal to LastEvaluationTime + resendDelay, send again
 	return a.LastSentAt.Add(resendDelay).Before(a.LastEvaluationTime) ||
 		a.LastSentAt.Add(resendDelay).Equal(a.LastEvaluationTime)

pkg/services/ngalert/state/state_test.go

@@ -67,6 +67,39 @@ func TestNeedsSending(t *testing.T) {
 				LastSentAt:         evaluationTime,
 			},
 		},
+		{
+			name:        "state: normal + resolved sends after a minute",
+			resendDelay: 1 * time.Minute,
+			expected:    true,
+			testState: &State{
+				State:              eval.Normal,
+				Resolved:           true,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime.Add(-1 * time.Minute),
+			},
+		},
+		{
+			name:        "state: normal + resolved does _not_ send after 30 seconds (before one minute)",
+			resendDelay: 1 * time.Minute,
+			expected:    false,
+			testState: &State{
+				State:              eval.Normal,
+				Resolved:           true,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime.Add(-30 * time.Second),
+			},
+		},
+		{
+			name:        "state: normal but not resolved does not send after a minute",
+			resendDelay: 1 * time.Minute,
+			expected:    false,
+			testState: &State{
+				State:              eval.Normal,
+				Resolved:           false,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime.Add(-1 * time.Minute),
+			},
+		},
 	}
 
 	for _, tc := range testCases {