Alerting resend delay for sending to notifiers (#34312)

* adds resend delay to avoid saturating notifier

* correct method signatures

* pr feedback

pkg/services/ngalert/state/manager.go

@@ -11,18 +11,20 @@ import (
 )
 
 type Manager struct {
-	cache   *cache
-	quit    chan struct{}
-	Log     log.Logger
-	metrics *metrics.Metrics
+	cache       *cache
+	quit        chan struct{}
+	ResendDelay time.Duration
+	Log         log.Logger
+	metrics     *metrics.Metrics
 }
 
 func NewManager(logger log.Logger, metrics *metrics.Metrics) *Manager {
 	manager := &Manager{
-		cache:   newCache(logger, metrics),
-		quit:    make(chan struct{}),
-		Log:     logger,
-		metrics: metrics,
+		cache:       newCache(logger, metrics),
+		quit:        make(chan struct{}),
+		ResendDelay: 1 * time.Minute, // TODO: make this configurable
+		Log:         logger,
+		metrics:     metrics,
 	}
 	go manager.recordMetrics()
 	return manager
@@ -65,7 +67,6 @@ func (st *Manager) ProcessEvalResults(alertRule *ngModels.AlertRule, results eva
 	return states
 }
 
-//TODO: When calculating if an alert should not be firing anymore, we should take into account the re-send delay if any. We don't want to send every firing alert every time, we should have a fixed delay across all alerts to avoid saturating the notification system
 //Set the current state based on evaluation results
 func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Result) *State {
 	currentState := st.getOrCreate(alertRule, result)

pkg/services/ngalert/state/state.go

@@ -12,14 +12,15 @@ type State struct {
 	AlertRuleUID       string
 	OrgID              int64
 	CacheId            string
-	Labels             data.Labels
 	State              eval.State
 	Results            []Evaluation
 	StartsAt           time.Time
 	EndsAt             time.Time
 	LastEvaluationTime time.Time
 	EvaluationDuration time.Duration
+	LastSentAt         time.Time
 	Annotations        map[string]string
+	Labels             data.Labels
 	Error              error
 }
 
@@ -112,6 +113,16 @@ func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result)
 	return a
 }
 
+func (a *State) NeedsSending(resendDelay time.Duration) bool {
+	if a.State != eval.Alerting {
+		return false
+	}
+
+	// if LastSentAt is before or equal to LastEvaluationTime + resendDelay, send again
+	return a.LastSentAt.Add(resendDelay).Before(a.LastEvaluationTime) ||
+		a.LastSentAt.Add(resendDelay).Equal(a.LastEvaluationTime)
+}
+
 func (a *State) Equals(b *State) bool {
 	return a.AlertRuleUID == b.AlertRuleUID &&
 		a.OrgID == b.OrgID &&

pkg/services/ngalert/state/state_test.go

@@ -0,0 +1,75 @@
+package state
+
+import (
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNeedsSending(t *testing.T) {
+	evaluationTime, _ := time.Parse("2006-01-02", "2021-03-25")
+	testCases := []struct {
+		name        string
+		resendDelay time.Duration
+		expected    bool
+		testState   *State
+	}{
+		{
+			name:        "state: alerting and LastSentAt before LastEvaluationTime + ResendDelay",
+			resendDelay: 1 * time.Minute,
+			expected:    true,
+			testState: &State{
+				State:              eval.Alerting,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime.Add(-2 * time.Minute),
+			},
+		},
+		{
+			name:        "state: alerting and LastSentAt after LastEvaluationTime + ResendDelay",
+			resendDelay: 1 * time.Minute,
+			expected:    false,
+			testState: &State{
+				State:              eval.Alerting,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime,
+			},
+		},
+		{
+			name:        "state: alerting and LastSentAt equals LastEvaluationTime + ResendDelay",
+			resendDelay: 1 * time.Minute,
+			expected:    true,
+			testState: &State{
+				State:              eval.Alerting,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime.Add(-1 * time.Minute),
+			},
+		},
+		{
+			name:        "state: pending",
+			resendDelay: 1 * time.Minute,
+			expected:    false,
+			testState: &State{
+				State: eval.Pending,
+			},
+		},
+		{
+			name:        "state: alerting and ResendDelay is zero",
+			resendDelay: 0 * time.Minute,
+			expected:    true,
+			testState: &State{
+				State:              eval.Alerting,
+				LastEvaluationTime: evaluationTime,
+				LastSentAt:         evaluationTime,
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expected, tc.testState.NeedsSending(tc.resendDelay))
+		})
+	}
+}