diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index 9f7bba3bc1a..4293f92dd18 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -440,7 +440,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -480,7 +480,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -525,7 +525,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Alerting), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -571,7 +571,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -587,7 +587,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), // TODO probably it should be t1 (semantic of Normal)? + EndsAt: t1.Add(ResendDelay * 4), // TODO probably it should be t1 (semantic of Normal)? LastEvaluationTime: t2, }, }, @@ -603,7 +603,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -796,7 +796,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -822,7 +822,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -852,7 +852,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Alerting), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -942,7 +942,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.NoData), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -960,7 +960,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.NoData), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -1008,7 +1008,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1026,7 +1026,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1065,7 +1065,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1118,7 +1118,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1163,7 +1163,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1212,7 +1212,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1282,7 +1282,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1297,7 +1297,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t1, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1316,7 +1316,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1333,7 +1333,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t1, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1439,7 +1439,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1482,7 +1482,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1500,7 +1500,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1545,7 +1545,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1612,7 +1612,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1626,7 +1626,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1643,7 +1643,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1658,7 +1658,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1757,7 +1757,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1775,7 +1775,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1793,7 +1793,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1814,7 +1814,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1833,7 +1833,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -1978,7 +1978,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -1996,7 +1996,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2035,7 +2035,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2087,7 +2087,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2118,7 +2118,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -2153,7 +2153,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t2, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -2209,7 +2209,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t1, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2228,7 +2228,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.NoData), }, StartsAt: t1, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -2301,7 +2301,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2317,7 +2317,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -2335,7 +2335,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -2353,7 +2353,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -2374,7 +2374,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -2393,7 +2393,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -2491,7 +2491,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Error), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, Annotations: mergeLabels(baseRule.Annotations, data.Labels{ "Error": datasourceError.Error(), @@ -2513,7 +2513,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Error), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -2559,7 +2559,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Error), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, Annotations: mergeLabels(baseRule.Annotations, data.Labels{ "Error": genericError.Error(), @@ -2581,7 +2581,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Error), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -2632,7 +2632,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, Annotations: mergeLabels(baseRule.Annotations, data.Labels{ "Error": datasourceError.Error(), @@ -2654,7 +2654,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2693,7 +2693,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2743,7 +2743,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, Annotations: mergeLabels(baseRule.Annotations, data.Labels{ "Error": datasourceError.Error(), @@ -2765,7 +2765,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2805,7 +2805,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -2978,7 +2978,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, Annotations: mergeLabels(baseRule.Annotations, data.Labels{ "Error": datasourceError.Error(), @@ -3001,7 +3001,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -3051,7 +3051,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t1, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), + EndsAt: t1.Add(ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -3068,7 +3068,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, Annotations: mergeLabels(baseRule.Annotations, data.Labels{ "Error": datasourceError.Error(), @@ -3090,7 +3090,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -3145,7 +3145,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(ResendDelay * 3), + EndsAt: t2.Add(ResendDelay * 4), LastEvaluationTime: t2, Annotations: mergeLabels(baseRule.Annotations, data.Labels{ "Error": datasourceError.Error(), @@ -3164,7 +3164,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -3184,7 +3184,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 3), // TODO probably it should be t1 (semantic of Normal)? + EndsAt: t1.Add(ResendDelay * 4), // TODO probably it should be t1 (semantic of Normal)? LastEvaluationTime: t2, }, }, @@ -3201,7 +3201,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, @@ -3237,7 +3237,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(ResendDelay * 3), + EndsAt: t3.Add(ResendDelay * 4), LastEvaluationTime: t3, }, }, diff --git a/pkg/services/ngalert/state/manager_test.go b/pkg/services/ngalert/state/manager_test.go index 682af1bcb53..e24934a6376 100644 --- a/pkg/services/ngalert/state/manager_test.go +++ b/pkg/services/ngalert/state/manager_test.go @@ -423,7 +423,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t1, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(state.ResendDelay * 3), + EndsAt: t1.Add(state.ResendDelay * 4), LastEvaluationTime: t1, }, }, @@ -474,7 +474,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.Alerting), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -506,7 +506,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(tn(4), eval.Alerting), }, StartsAt: tn(4), - EndsAt: tn(4).Add(state.ResendDelay * 3), + EndsAt: tn(4).Add(state.ResendDelay * 4), LastEvaluationTime: tn(4), }, }, @@ -541,7 +541,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(tn(5), eval.Alerting), }, StartsAt: tn(4), - EndsAt: tn(4).Add(state.ResendDelay * 3), + EndsAt: tn(4).Add(state.ResendDelay * 4), LastEvaluationTime: tn(5), }, }, @@ -573,7 +573,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(tn(4), eval.NoData), }, StartsAt: tn(4), - EndsAt: tn(4).Add(state.ResendDelay * 3), + EndsAt: tn(4).Add(state.ResendDelay * 4), LastEvaluationTime: tn(4), }, }, @@ -600,7 +600,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.Alerting), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -626,7 +626,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.Alerting), }, StartsAt: t1, - EndsAt: t1.Add(state.ResendDelay * 3), + EndsAt: t1.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -653,7 +653,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -690,7 +690,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(tn(5), eval.NoData), }, StartsAt: tn(5), - EndsAt: tn(5).Add(state.ResendDelay * 3), + EndsAt: tn(5).Add(state.ResendDelay * 4), LastEvaluationTime: tn(5), }, }, @@ -716,7 +716,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -751,7 +751,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -797,7 +797,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -836,7 +836,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.NoData), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -891,7 +891,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, }, }, @@ -929,7 +929,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(tn(5), eval.Error), }, StartsAt: tn(5), - EndsAt: tn(5).Add(state.ResendDelay * 3), + EndsAt: tn(5).Add(state.ResendDelay * 4), LastEvaluationTime: tn(5), }, }, @@ -967,7 +967,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t2, eval.Error), }, StartsAt: t2, - EndsAt: t2.Add(state.ResendDelay * 3), + EndsAt: t2.Add(state.ResendDelay * 4), LastEvaluationTime: t2, EvaluationDuration: evaluationDuration, Annotations: map[string]string{"annotation": "test", "Error": "[sse.dataQueryError] failed to execute query [A]: this is an error"}, @@ -1062,7 +1062,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(tn(6), eval.Error), }, StartsAt: tn(4), - EndsAt: tn(6).Add(state.ResendDelay * 3), + EndsAt: tn(6).Add(state.ResendDelay * 4), LastEvaluationTime: tn(6), }, }, @@ -1095,7 +1095,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(tn(8), eval.Alerting), }, StartsAt: tn(8), - EndsAt: tn(8).Add(state.ResendDelay * 3), + EndsAt: tn(8).Add(state.ResendDelay * 4), LastEvaluationTime: tn(8), }, }, @@ -1128,7 +1128,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(tn(6), eval.NoData), }, StartsAt: tn(6), - EndsAt: tn(6).Add(state.ResendDelay * 3), + EndsAt: tn(6).Add(state.ResendDelay * 4), LastEvaluationTime: tn(6), }, }, @@ -1194,7 +1194,7 @@ func TestProcessEvalResults(t *testing.T) { newEvaluation(t3, eval.Alerting), }, StartsAt: t3, - EndsAt: t3.Add(state.ResendDelay * 3), + EndsAt: t3.Add(state.ResendDelay * 4), LastEvaluationTime: t3, }, }, diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index f7389112bd9..89ad132cac6 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -410,7 +410,10 @@ func nextEndsTime(interval int64, evaluatedAt time.Time) time.Time { if intv > ResendDelay { ends = intv } - return evaluatedAt.Add(3 * ends) + // Allow for at least two evaluation cycles to pass before expiring, every time. + // Synchronized with Prometheus: + // https://github.com/prometheus/prometheus/blob/6a9b3263ffdba5ea8c23e6f9ef69fb7a15b566f8/rules/alerting.go#L493 + return evaluatedAt.Add(4 * ends) } func (a *State) GetLabels(opts ...models.LabelOption) map[string]string { diff --git a/pkg/services/ngalert/state/state_test.go b/pkg/services/ngalert/state/state_test.go index 9d4684e45bc..03e72d76e31 100644 --- a/pkg/services/ngalert/state/state_test.go +++ b/pkg/services/ngalert/state/state_test.go @@ -253,14 +253,14 @@ func TestMaintain(t *testing.T) { // the interval is less than the resend interval of 30 seconds s := State{State: eval.Alerting, StartsAt: now, EndsAt: now.Add(time.Second)} s.Maintain(10, now.Add(10*time.Second)) - // 10 seconds + 3 x 30 seconds is 100 seconds - assert.Equal(t, now.Add(100*time.Second), s.EndsAt) + // 10 seconds + 4 x 30 seconds is 130 seconds + assert.Equal(t, now.Add(130*time.Second), s.EndsAt) // the interval is above the resend interval of 30 seconds s = State{State: eval.Alerting, StartsAt: now, EndsAt: now.Add(time.Second)} s.Maintain(60, now.Add(10*time.Second)) - // 10 seconds + 3 x 60 seconds is 190 seconds - assert.Equal(t, now.Add(190*time.Second), s.EndsAt) + // 10 seconds + 4 x 60 seconds is 250 seconds + assert.Equal(t, now.Add(250*time.Second), s.EndsAt) } func TestEnd(t *testing.T) { @@ -273,14 +273,14 @@ func TestEnd(t *testing.T) { }{ { name: "less than resend delay: for=unset,interval=10s - endsAt = resendDelay * 3", - expected: evaluationTime.Add(ResendDelay * 3), + expected: evaluationTime.Add(ResendDelay * 4), testRule: &ngmodels.AlertRule{ IntervalSeconds: 10, }, }, { name: "less than resend delay: for=0s,interval=10s - endsAt = resendDelay * 3", - expected: evaluationTime.Add(ResendDelay * 3), + expected: evaluationTime.Add(ResendDelay * 4), testRule: &ngmodels.AlertRule{ For: 0 * time.Second, IntervalSeconds: 10, @@ -288,7 +288,7 @@ func TestEnd(t *testing.T) { }, { name: "less than resend delay: for=10s,interval=10s - endsAt = resendDelay * 3", - expected: evaluationTime.Add(ResendDelay * 3), + expected: evaluationTime.Add(ResendDelay * 4), testRule: &ngmodels.AlertRule{ For: 10 * time.Second, IntervalSeconds: 10, @@ -296,7 +296,7 @@ func TestEnd(t *testing.T) { }, { name: "less than resend delay: for=10s,interval=20s - endsAt = resendDelay * 3", - expected: evaluationTime.Add(ResendDelay * 3), + expected: evaluationTime.Add(ResendDelay * 4), testRule: &ngmodels.AlertRule{ For: 10 * time.Second, IntervalSeconds: 20, @@ -304,14 +304,14 @@ func TestEnd(t *testing.T) { }, { name: "more than resend delay: for=unset,interval=1m - endsAt = interval * 3", - expected: evaluationTime.Add(time.Second * 60 * 3), + expected: evaluationTime.Add(time.Second * 60 * 4), testRule: &ngmodels.AlertRule{ IntervalSeconds: 60, }, }, { name: "more than resend delay: for=0s,interval=1m - endsAt = resendDelay * 3", - expected: evaluationTime.Add(time.Second * 60 * 3), + expected: evaluationTime.Add(time.Second * 60 * 4), testRule: &ngmodels.AlertRule{ For: 0 * time.Second, IntervalSeconds: 60, @@ -319,7 +319,7 @@ func TestEnd(t *testing.T) { }, { name: "more than resend delay: for=1m,interval=5m - endsAt = interval * 3", - expected: evaluationTime.Add(time.Second * 300 * 3), + expected: evaluationTime.Add(time.Second * 300 * 4), testRule: &ngmodels.AlertRule{ For: time.Minute, IntervalSeconds: 300, @@ -327,7 +327,7 @@ func TestEnd(t *testing.T) { }, { name: "more than resend delay: for=5m,interval=1m - endsAt = interval * 3", - expected: evaluationTime.Add(time.Second * 60 * 3), + expected: evaluationTime.Add(time.Second * 60 * 4), testRule: &ngmodels.AlertRule{ For: 300 * time.Second, IntervalSeconds: 60,