NGAlert: Send resolve signal to alertmanager on alerting -> Normal (#37363)

pkg/services/ngalert/schedule/compat.go

@@ -28,34 +28,35 @@ func FromAlertStateToPostableAlerts(logger log.Logger, firingStates []*state.Sta
 	}
 
 	for _, alertState := range firingStates {
-		if alertState.NeedsSending(stateManager.ResendDelay) {
+		if !alertState.NeedsSending(stateManager.ResendDelay) {
-			nL := alertState.Labels.Copy()
+			continue
-			nA := data.Labels(alertState.Annotations).Copy()
-
-			if len(alertState.Results) > 0 {
-				nA["__value_string__"] = alertState.Results[0].EvaluationString
-			}
-
-			genURL := appURL
-			if uid := nL[ngModels.RuleUIDLabel]; len(uid) > 0 && u != nil {
-				oldPath := u.Path
-				u.Path = path.Join(u.Path, fmt.Sprintf("/alerting/%s/edit", uid))
-				genURL = u.String()
-				u.Path = oldPath
-			}
-
-			alerts.PostableAlerts = append(alerts.PostableAlerts, models.PostableAlert{
-				Annotations: models.LabelSet(nA),
-				StartsAt:    strfmt.DateTime(alertState.StartsAt),
-				EndsAt:      strfmt.DateTime(alertState.EndsAt),
-				Alert: models.Alert{
-					Labels:       models.LabelSet(nL),
-					GeneratorURL: strfmt.URI(genURL),
-				},
-			})
-			alertState.LastSentAt = ts
-			sentAlerts = append(sentAlerts, alertState)
 		}
+		nL := alertState.Labels.Copy()
+		nA := data.Labels(alertState.Annotations).Copy()
+
+		if len(alertState.Results) > 0 {
+			nA["__value_string__"] = alertState.Results[0].EvaluationString
+		}
+
+		genURL := appURL
+		if uid := nL[ngModels.RuleUIDLabel]; len(uid) > 0 && u != nil {
+			oldPath := u.Path
+			u.Path = path.Join(u.Path, fmt.Sprintf("/alerting/%s/edit", uid))
+			genURL = u.String()
+			u.Path = oldPath
+		}
+
+		alerts.PostableAlerts = append(alerts.PostableAlerts, models.PostableAlert{
+			Annotations: models.LabelSet(nA),
+			StartsAt:    strfmt.DateTime(alertState.StartsAt),
+			EndsAt:      strfmt.DateTime(alertState.EndsAt),
+			Alert: models.Alert{
+				Labels:       models.LabelSet(nL),
+				GeneratorURL: strfmt.URI(genURL),
+			},
+		})
+		alertState.LastSentAt = ts
+		sentAlerts = append(sentAlerts, alertState)
 	}
 	stateManager.Put(sentAlerts)
 	return alerts

pkg/services/ngalert/state/manager.go

@@ -175,6 +175,10 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul
 	case eval.Pending: // we do not emit results with this state
 	}
 
+	// Set Resolved property so the scheduler knows to send a postable alert
+	// to Alertmanager.
+	currentState.Resolved = oldState == eval.Alerting && currentState.State == eval.Normal
+
 	st.set(currentState)
 	if oldState != currentState.State {
 		go st.createAlertAnnotation(currentState.State, alertRule, result)

pkg/services/ngalert/state/state.go

@@ -13,6 +13,7 @@ type State struct {
 	OrgID              int64
 	CacheId            string
 	State              eval.State
+	Resolved           bool
 	Results            []Evaluation
 	StartsAt           time.Time
 	EndsAt             time.Time
@@ -112,10 +113,13 @@ func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result)
 }
 
 func (a *State) NeedsSending(resendDelay time.Duration) bool {
-	if a.State != eval.Alerting {
+	if a.State != eval.Alerting && a.State != eval.Normal {
 		return false
 	}
 
+	if a.State == eval.Normal && !a.Resolved {
+		return false
+	}
 	// if LastSentAt is before or equal to LastEvaluationTime + resendDelay, send again
 	return a.LastSentAt.Add(resendDelay).Before(a.LastEvaluationTime) ||
 		a.LastSentAt.Add(resendDelay).Equal(a.LastEvaluationTime)

pkg/services/ngalert/state/state_test.go

@@ -67,6 +67,39 @@ func TestNeedsSending(t *testing.T) {
 				LastSentAt:         evaluationTime,
 			},
 		},
+		{
+			name:        "state: normal + resolved sends after a minute",
+			resendDelay: 1 * time.Minute,
+			expected:    true,
+			testState: &State{
+				State:              eval.Normal,
+				Resolved:           true,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime.Add(-1 * time.Minute),
+			},
+		},
+		{
+			name:        "state: normal + resolved does _not_ send after 30 seconds (before one minute)",
+			resendDelay: 1 * time.Minute,
+			expected:    false,
+			testState: &State{
+				State:              eval.Normal,
+				Resolved:           true,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime.Add(-30 * time.Second),
+			},
+		},
+		{
+			name:        "state: normal but not resolved does not send after a minute",
+			resendDelay: 1 * time.Minute,
+			expected:    false,
+			testState: &State{
+				State:              eval.Normal,
+				Resolved:           false,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime.Add(-1 * time.Minute),
+			},
+		},
 	}
 
 	for _, tc := range testCases {