NGAlert: Send resolve signal to alertmanager on alerting -> Normal (#37363)

This commit is contained in:
Kyle Brandt 2021-07-29 14:29:17 -04:00 committed by GitHub
parent c8216244a2
commit aa904a5a04
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 70 additions and 28 deletions

View File

@ -28,7 +28,9 @@ func FromAlertStateToPostableAlerts(logger log.Logger, firingStates []*state.Sta
}
for _, alertState := range firingStates {
if alertState.NeedsSending(stateManager.ResendDelay) {
if !alertState.NeedsSending(stateManager.ResendDelay) {
continue
}
nL := alertState.Labels.Copy()
nA := data.Labels(alertState.Annotations).Copy()
@ -56,7 +58,6 @@ func FromAlertStateToPostableAlerts(logger log.Logger, firingStates []*state.Sta
alertState.LastSentAt = ts
sentAlerts = append(sentAlerts, alertState)
}
}
stateManager.Put(sentAlerts)
return alerts
}

View File

@ -175,6 +175,10 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul
case eval.Pending: // we do not emit results with this state
}
// Set Resolved property so the scheduler knows to send a postable alert
// to Alertmanager.
currentState.Resolved = oldState == eval.Alerting && currentState.State == eval.Normal
st.set(currentState)
if oldState != currentState.State {
go st.createAlertAnnotation(currentState.State, alertRule, result)

View File

@ -13,6 +13,7 @@ type State struct {
OrgID int64
CacheId string
State eval.State
Resolved bool
Results []Evaluation
StartsAt time.Time
EndsAt time.Time
@ -112,10 +113,13 @@ func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result)
}
func (a *State) NeedsSending(resendDelay time.Duration) bool {
if a.State != eval.Alerting {
if a.State != eval.Alerting && a.State != eval.Normal {
return false
}
if a.State == eval.Normal && !a.Resolved {
return false
}
// if LastSentAt is before or equal to LastEvaluationTime + resendDelay, send again
return a.LastSentAt.Add(resendDelay).Before(a.LastEvaluationTime) ||
a.LastSentAt.Add(resendDelay).Equal(a.LastEvaluationTime)

View File

@ -67,6 +67,39 @@ func TestNeedsSending(t *testing.T) {
LastSentAt: evaluationTime,
},
},
{
name: "state: normal + resolved sends after a minute",
resendDelay: 1 * time.Minute,
expected: true,
testState: &State{
State: eval.Normal,
Resolved: true,
LastEvaluationTime: evaluationTime,
LastSentAt: evaluationTime.Add(-1 * time.Minute),
},
},
{
name: "state: normal + resolved does _not_ send after 30 seconds (before one minute)",
resendDelay: 1 * time.Minute,
expected: false,
testState: &State{
State: eval.Normal,
Resolved: true,
LastEvaluationTime: evaluationTime,
LastSentAt: evaluationTime.Add(-30 * time.Second),
},
},
{
name: "state: normal but not resolved does not send after a minute",
resendDelay: 1 * time.Minute,
expected: false,
testState: &State{
State: eval.Normal,
Resolved: false,
LastEvaluationTime: evaluationTime,
LastSentAt: evaluationTime.Add(-1 * time.Minute),
},
},
}
for _, tc := range testCases {