IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Allow more time before Alertmanager expire-resolves alerts (#77094)

Browse Source 
* Sync endsAt factor with prometheus

* Fix state tests
This commit is contained in:
Alexander Weaver
2023-10-25 10:03:46 -05:00
committed by GitHub
parent 4fc0294aab
commit 6ee52ac80c
4 changed files with 104 additions and 101 deletions
Download Patch File Download Diff File Expand all files Collapse all files

136
pkg/services/ngalert/state/manager_private_test.go
View File

@@ -440,7 +440,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -480,7 +480,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -525,7 +525,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Alerting),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -571,7 +571,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -587,7 +587,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3), // TODO probably it should be t1 (semantic of Normal)?
EndsAt: t1.Add(ResendDelay * 4), // TODO probably it should be t1 (semantic of Normal)?
LastEvaluationTime: t2,
},
},
@@ -603,7 +603,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -796,7 +796,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -822,7 +822,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -852,7 +852,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Alerting),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -942,7 +942,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.NoData),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -960,7 +960,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.NoData),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -1008,7 +1008,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1026,7 +1026,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1065,7 +1065,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1118,7 +1118,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1163,7 +1163,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1212,7 +1212,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1282,7 +1282,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1297,7 +1297,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1316,7 +1316,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1333,7 +1333,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1439,7 +1439,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1482,7 +1482,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1500,7 +1500,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1545,7 +1545,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1612,7 +1612,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1626,7 +1626,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1643,7 +1643,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1658,7 +1658,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1757,7 +1757,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1775,7 +1775,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1793,7 +1793,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1814,7 +1814,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1833,7 +1833,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -1978,7 +1978,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -1996,7 +1996,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2035,7 +2035,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2087,7 +2087,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2118,7 +2118,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -2153,7 +2153,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t2,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -2209,7 +2209,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t1,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2228,7 +2228,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.NoData),
},
StartsAt: t1,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -2301,7 +2301,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2317,7 +2317,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -2335,7 +2335,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -2353,7 +2353,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -2374,7 +2374,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -2393,7 +2393,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -2491,7 +2491,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Error),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
@@ -2513,7 +2513,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Error),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -2559,7 +2559,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Error),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": genericError.Error(),
@@ -2581,7 +2581,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Error),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -2632,7 +2632,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
@@ -2654,7 +2654,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2693,7 +2693,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2743,7 +2743,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
@@ -2765,7 +2765,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2805,7 +2805,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -2978,7 +2978,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
@@ -3001,7 +3001,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -3051,7 +3051,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3),
EndsAt: t1.Add(ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -3068,7 +3068,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
@@ -3090,7 +3090,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -3145,7 +3145,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(ResendDelay * 3),
EndsAt: t2.Add(ResendDelay * 4),
LastEvaluationTime: t2,
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
"Error": datasourceError.Error(),
@@ -3164,7 +3164,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -3184,7 +3184,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t1,
EndsAt: t1.Add(ResendDelay * 3), // TODO probably it should be t1 (semantic of Normal)?
EndsAt: t1.Add(ResendDelay * 4), // TODO probably it should be t1 (semantic of Normal)?
LastEvaluationTime: t2,
},
},
@@ -3201,7 +3201,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},
@@ -3237,7 +3237,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(ResendDelay * 3),
EndsAt: t3.Add(ResendDelay * 4),
LastEvaluationTime: t3,
},
},

40
pkg/services/ngalert/state/manager_test.go
View File

@@ -423,7 +423,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 3),
EndsAt: t1.Add(state.ResendDelay * 4),
LastEvaluationTime: t1,
},
},
@@ -474,7 +474,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.Alerting),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -506,7 +506,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(tn(4), eval.Alerting),
},
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 3),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
},
},
@@ -541,7 +541,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(tn(5), eval.Alerting),
},
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 3),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
},
},
@@ -573,7 +573,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(tn(4), eval.NoData),
},
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 3),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
},
},
@@ -600,7 +600,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.Alerting),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -626,7 +626,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.Alerting),
},
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 3),
EndsAt: t1.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -653,7 +653,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -690,7 +690,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(tn(5), eval.NoData),
},
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 3),
EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
},
},
@@ -716,7 +716,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -751,7 +751,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -797,7 +797,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -836,7 +836,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.NoData),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -891,7 +891,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
},
},
@@ -929,7 +929,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(tn(5), eval.Error),
},
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 3),
EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
},
},
@@ -967,7 +967,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t2, eval.Error),
},
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 3),
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "Error": "[sse.dataQueryError] failed to execute query [A]: this is an error"},
@@ -1062,7 +1062,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(tn(6), eval.Error),
},
StartsAt: tn(4),
EndsAt: tn(6).Add(state.ResendDelay * 3),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
},
},
@@ -1095,7 +1095,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(tn(8), eval.Alerting),
},
StartsAt: tn(8),
EndsAt: tn(8).Add(state.ResendDelay * 3),
EndsAt: tn(8).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(8),
},
},
@@ -1128,7 +1128,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(tn(6), eval.NoData),
},
StartsAt: tn(6),
EndsAt: tn(6).Add(state.ResendDelay * 3),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
},
},
@@ -1194,7 +1194,7 @@ func TestProcessEvalResults(t *testing.T) {
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3,
EndsAt: t3.Add(state.ResendDelay * 3),
EndsAt: t3.Add(state.ResendDelay * 4),
LastEvaluationTime: t3,
},
},

5
pkg/services/ngalert/state/state.go
View File

@@ -410,7 +410,10 @@ func nextEndsTime(interval int64, evaluatedAt time.Time) time.Time {
if intv > ResendDelay {
ends = intv
}
return evaluatedAt.Add(3 * ends)
// Allow for at least two evaluation cycles to pass before expiring, every time.
// Synchronized with Prometheus:
// https://github.com/prometheus/prometheus/blob/6a9b3263ffdba5ea8c23e6f9ef69fb7a15b566f8/rules/alerting.go#L493
return evaluatedAt.Add(4 * ends)
}
func (a *State) GetLabels(opts ...models.LabelOption) map[string]string {

24
pkg/services/ngalert/state/state_test.go
View File

@@ -253,14 +253,14 @@ func TestMaintain(t *testing.T) {
// the interval is less than the resend interval of 30 seconds
s := State{State: eval.Alerting, StartsAt: now, EndsAt: now.Add(time.Second)}
s.Maintain(10, now.Add(10*time.Second))
// 10 seconds + 3 x 30 seconds is 100 seconds
assert.Equal(t, now.Add(100*time.Second), s.EndsAt)
// 10 seconds + 4 x 30 seconds is 130 seconds
assert.Equal(t, now.Add(130*time.Second), s.EndsAt)
// the interval is above the resend interval of 30 seconds
s = State{State: eval.Alerting, StartsAt: now, EndsAt: now.Add(time.Second)}
s.Maintain(60, now.Add(10*time.Second))
// 10 seconds + 3 x 60 seconds is 190 seconds
assert.Equal(t, now.Add(190*time.Second), s.EndsAt)
// 10 seconds + 4 x 60 seconds is 250 seconds
assert.Equal(t, now.Add(250*time.Second), s.EndsAt)
}
func TestEnd(t *testing.T) {
@@ -273,14 +273,14 @@ func TestEnd(t *testing.T) {
}{
{
name: "less than resend delay: for=unset,interval=10s - endsAt = resendDelay * 3",
expected: evaluationTime.Add(ResendDelay * 3),
expected: evaluationTime.Add(ResendDelay * 4),
testRule: &ngmodels.AlertRule{
IntervalSeconds: 10,
},
},
{
name: "less than resend delay: for=0s,interval=10s - endsAt = resendDelay * 3",
expected: evaluationTime.Add(ResendDelay * 3),
expected: evaluationTime.Add(ResendDelay * 4),
testRule: &ngmodels.AlertRule{
For: 0 * time.Second,
IntervalSeconds: 10,
@@ -288,7 +288,7 @@ func TestEnd(t *testing.T) {
},
{
name: "less than resend delay: for=10s,interval=10s - endsAt = resendDelay * 3",
expected: evaluationTime.Add(ResendDelay * 3),
expected: evaluationTime.Add(ResendDelay * 4),
testRule: &ngmodels.AlertRule{
For: 10 * time.Second,
IntervalSeconds: 10,
@@ -296,7 +296,7 @@ func TestEnd(t *testing.T) {
},
{
name: "less than resend delay: for=10s,interval=20s - endsAt = resendDelay * 3",
expected: evaluationTime.Add(ResendDelay * 3),
expected: evaluationTime.Add(ResendDelay * 4),
testRule: &ngmodels.AlertRule{
For: 10 * time.Second,
IntervalSeconds: 20,
@@ -304,14 +304,14 @@ func TestEnd(t *testing.T) {
},
{
name: "more than resend delay: for=unset,interval=1m - endsAt = interval * 3",
expected: evaluationTime.Add(time.Second * 60 * 3),
expected: evaluationTime.Add(time.Second * 60 * 4),
testRule: &ngmodels.AlertRule{
IntervalSeconds: 60,
},
},
{
name: "more than resend delay: for=0s,interval=1m - endsAt = resendDelay * 3",
expected: evaluationTime.Add(time.Second * 60 * 3),
expected: evaluationTime.Add(time.Second * 60 * 4),
testRule: &ngmodels.AlertRule{
For: 0 * time.Second,
IntervalSeconds: 60,
@@ -319,7 +319,7 @@ func TestEnd(t *testing.T) {
},
{
name: "more than resend delay: for=1m,interval=5m - endsAt = interval * 3",
expected: evaluationTime.Add(time.Second * 300 * 3),
expected: evaluationTime.Add(time.Second * 300 * 4),
testRule: &ngmodels.AlertRule{
For: time.Minute,
IntervalSeconds: 300,
@@ -327,7 +327,7 @@ func TestEnd(t *testing.T) {
},
{
name: "more than resend delay: for=5m,interval=1m - endsAt = interval * 3",
expected: evaluationTime.Add(time.Second * 60 * 3),
expected: evaluationTime.Add(time.Second * 60 * 4),
testRule: &ngmodels.AlertRule{
For: 300 * time.Second,
IntervalSeconds: 60,