IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Do not store series values from past evaluations in state manager for no reason (#87525)

Browse Source 
Do not store previous execution results on states
This commit is contained in:
Alexander Weaver 2024-05-09 15:51:55 -05:00 committed by GitHub
parent c6a0175c04
commit a6a9ab4008
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 902 additions and 1512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pkg/services/ngalert/api/api_prometheus_test.go
View File

@ -45,9 +45,7 @@ func Test_FormatValues(t *testing.T) {
name: "with no value, it renders the evaluation string", name: "with no value, it renders the evaluation string",
alertState: &state.State{ alertState: &state.State{
LastEvaluationString: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]", LastEvaluationString: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
Results: []state.Evaluation{ LatestResult: &state.Evaluation{Condition: "A", Values: map[string]*float64{}},
{Condition: "A", Values: map[string]*float64{}},
},
}, },
expected: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]", expected: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
}, },
@ -55,9 +53,7 @@ func Test_FormatValues(t *testing.T) {
name: "with one value, it renders the single value", name: "with one value, it renders the single value",
alertState: &state.State{ alertState: &state.State{
LastEvaluationString: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]", LastEvaluationString: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
Results: []state.Evaluation{ LatestResult: &state.Evaluation{Condition: "A", Values: map[string]*float64{"A": &val1}},
{Condition: "A", Values: map[string]*float64{"A": &val1}},
},
}, },
expected: "1.1e+00", expected: "1.1e+00",
}, },
@ -65,9 +61,7 @@ func Test_FormatValues(t *testing.T) {
name: "with two values, it renders the value based on their refID and position", name: "with two values, it renders the value based on their refID and position",
alertState: &state.State{ alertState: &state.State{
LastEvaluationString: "[ var='B0' metric='vector(10) + time() % 50' labels={} value=1.1 ], [ var='B1' metric='vector(10) + time() % 50' labels={} value=1.4 ]", LastEvaluationString: "[ var='B0' metric='vector(10) + time() % 50' labels={} value=1.1 ], [ var='B1' metric='vector(10) + time() % 50' labels={} value=1.4 ]",
Results: []state.Evaluation{ LatestResult: &state.Evaluation{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2}},
{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2}},
},
}, },
expected: "B0: 1.1e+00, B1: 1.4e+00", expected: "B0: 1.1e+00, B1: 1.4e+00",
}, },
@ -75,9 +69,7 @@ func Test_FormatValues(t *testing.T) {
name: "with a high number of values, it renders the value based on their refID and position using a natural order", name: "with a high number of values, it renders the value based on their refID and position using a natural order",
alertState: &state.State{ alertState: &state.State{
LastEvaluationString: "[ var='B0' metric='vector(10) + time() % 50' labels={} value=1.1 ], [ var='B1' metric='vector(10) + time() % 50' labels={} value=1.4 ]", LastEvaluationString: "[ var='B0' metric='vector(10) + time() % 50' labels={} value=1.1 ], [ var='B1' metric='vector(10) + time() % 50' labels={} value=1.4 ]",
Results: []state.Evaluation{ LatestResult: &state.Evaluation{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2, "B2": &val1, "B10": &val2, "B11": &val1}},
{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2, "B2": &val1, "B10": &val2, "B11": &val1}},
},
}, },
expected: "B0: 1.1e+00, B10: 1.4e+00, B11: 1.1e+00, B1: 1.4e+00, B2: 1.1e+00", expected: "B0: 1.1e+00, B10: 1.4e+00, B11: 1.1e+00, B1: 1.4e+00, B2: 1.1e+00",
}, },
@ -246,12 +238,12 @@ func withAlertingState() forEachState {
return func(s *state.State) *state.State { return func(s *state.State) *state.State {
s.State = eval.Alerting s.State = eval.Alerting
value := float64(1.1) value := float64(1.1)
s.Results = append(s.Results, state.Evaluation{ s.LatestResult = &state.Evaluation{
EvaluationState: eval.Alerting, EvaluationState: eval.Alerting,
EvaluationTime: timeNow(), EvaluationTime: timeNow(),
Values: map[string]*float64{"B": &value}, Values: map[string]*float64{"B": &value},
Condition: "B", Condition: "B",
}) }
return s return s
} }
} }

9
pkg/services/ngalert/api/testing.go
View File

@ -83,18 +83,11 @@ func (f *fakeAlertInstanceManager) GenerateAlertInstances(orgID int64, alertRule
"instance_label": "test", "instance_label": "test",
}, },
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: &state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
{
EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationState: eval.Normal, EvaluationState: eval.Normal,
Values: make(map[string]*float64), Values: make(map[string]*float64),
}, },
},
LastEvaluationTime: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationDuration: evaluationDuration, EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"}, Annotations: map[string]string{"annotation": "test"},

6
pkg/services/ngalert/schedule/alert_rule_test.go
View File

@ -311,13 +311,13 @@ func TestRuleRoutine(t *testing.T) {
require.Len(t, states, 1) require.Len(t, states, 1)
s := states[0] s := states[0]
require.Equal(t, rule.UID, s.AlertRuleUID) require.Equal(t, rule.UID, s.AlertRuleUID)
require.Len(t, s.Results, 1) require.NotNil(t, s.LatestResult)
var expectedStatus = evalState var expectedStatus = evalState
if evalState == eval.Pending { if evalState == eval.Pending {
expectedStatus = eval.Alerting expectedStatus = eval.Alerting
} }
require.Equal(t, expectedStatus.String(), s.Results[0].EvaluationState.String()) require.Equal(t, expectedStatus.String(), s.LatestResult.EvaluationState.String())
require.Equal(t, expectedTime, s.Results[0].EvaluationTime) require.Equal(t, expectedTime, s.LatestResult.EvaluationTime)
}) })
t.Run("it should save alert instances to storage", func(t *testing.T) { t.Run("it should save alert instances to storage", func(t *testing.T) {
// TODO rewrite when we are able to mock/fake state manager // TODO rewrite when we are able to mock/fake state manager

5
pkg/services/ngalert/state/manager.go
View File

@ -358,14 +358,13 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
currentState.LastEvaluationTime = result.EvaluatedAt currentState.LastEvaluationTime = result.EvaluatedAt
currentState.EvaluationDuration = result.EvaluationDuration currentState.EvaluationDuration = result.EvaluationDuration
currentState.Results = append(currentState.Results, Evaluation{ currentState.LatestResult = &Evaluation{
EvaluationTime: result.EvaluatedAt, EvaluationTime: result.EvaluatedAt,
EvaluationState: result.State, EvaluationState: result.State,
Values: NewEvaluationValues(result.Values), Values: NewEvaluationValues(result.Values),
Condition: alertRule.Condition, Condition: alertRule.Condition,
}) }
currentState.LastEvaluationString = result.EvaluationString currentState.LastEvaluationString = result.EvaluationString
currentState.TrimResults(alertRule)
oldState := currentState.State oldState := currentState.State
oldReason := currentState.StateReason oldReason := currentState.StateReason

812
pkg/services/ngalert/state/manager_private_test.go
View File

File diff suppressed because it is too large Load Diff

204
pkg/services/ngalert/state/manager_test.go
View File

@ -57,9 +57,7 @@ func TestWarmStateCache(t *testing.T) {
OrgID: rule.OrgID, OrgID: rule.OrgID,
Labels: data.Labels{"test1": "testValue1"}, Labels: data.Labels{"test1": "testValue1"},
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
},
StartsAt: evaluationTime.Add(-1 * time.Minute), StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime, LastEvaluationTime: evaluationTime,
@ -70,9 +68,7 @@ func TestWarmStateCache(t *testing.T) {
OrgID: rule.OrgID, OrgID: rule.OrgID,
Labels: data.Labels{"test2": "testValue2"}, Labels: data.Labels{"test2": "testValue2"},
State: eval.Alerting, State: eval.Alerting,
Results: []state.Evaluation{ LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
},
StartsAt: evaluationTime.Add(-1 * time.Minute), StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime, LastEvaluationTime: evaluationTime,
@ -84,9 +80,7 @@ func TestWarmStateCache(t *testing.T) {
OrgID: rule.OrgID, OrgID: rule.OrgID,
Labels: data.Labels{"test3": "testValue3"}, Labels: data.Labels{"test3": "testValue3"},
State: eval.NoData, State: eval.NoData,
Results: []state.Evaluation{ LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.NoData},
{EvaluationTime: evaluationTime, EvaluationState: eval.NoData},
},
StartsAt: evaluationTime.Add(-1 * time.Minute), StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime, LastEvaluationTime: evaluationTime,
@ -98,9 +92,7 @@ func TestWarmStateCache(t *testing.T) {
OrgID: rule.OrgID, OrgID: rule.OrgID,
Labels: data.Labels{"test4": "testValue4"}, Labels: data.Labels{"test4": "testValue4"},
State: eval.Error, State: eval.Error,
Results: []state.Evaluation{ LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Error},
{EvaluationTime: evaluationTime, EvaluationState: eval.Error},
},
StartsAt: evaluationTime.Add(-1 * time.Minute), StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime, LastEvaluationTime: evaluationTime,
@ -112,9 +104,7 @@ func TestWarmStateCache(t *testing.T) {
OrgID: rule.OrgID, OrgID: rule.OrgID,
Labels: data.Labels{"test5": "testValue5"}, Labels: data.Labels{"test5": "testValue5"},
State: eval.Pending, State: eval.Pending,
Results: []state.Evaluation{ LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Pending},
{EvaluationTime: evaluationTime, EvaluationState: eval.Pending},
},
StartsAt: evaluationTime.Add(-1 * time.Minute), StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute), EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime, LastEvaluationTime: evaluationTime,
@ -226,7 +216,7 @@ func TestWarmStateCache(t *testing.T) {
setCacheID(entry) setCacheID(entry)
cacheEntry := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheID) cacheEntry := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheID)
if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "Results")); diff != "" { if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "LatestResult")); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff) t.Errorf("Result mismatch (-want +got):\n%s", diff)
t.FailNow() t.FailNow()
} }
@ -331,8 +321,8 @@ func TestProcessEvalResults(t *testing.T) {
ExecErrState: models.ErrorErrState, ExecErrState: models.ErrorErrState,
} }
newEvaluation := func(evalTime time.Time, evalState eval.State) state.Evaluation { newEvaluation := func(evalTime time.Time, evalState eval.State) *state.Evaluation {
return state.Evaluation{ return &state.Evaluation{
EvaluationTime: evalTime, EvaluationTime: evalTime,
EvaluationState: evalState, EvaluationState: evalState,
Values: make(map[string]*float64), Values: make(map[string]*float64),
@ -398,9 +388,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: newEvaluation(t1, eval.Normal),
newEvaluation(t1, eval.Normal),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t1, LastEvaluationTime: t1,
@ -422,9 +410,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: newEvaluation(t1, eval.Normal),
newEvaluation(t1, eval.Normal),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t1, LastEvaluationTime: t1,
@ -433,9 +419,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels2"], Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(), ResultFingerprint: labels2.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
Results: []state.Evaluation{ LatestResult: newEvaluation(t1, eval.Alerting),
newEvaluation(t1, eval.Alerting),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4), EndsAt: t1.Add(state.ResendDelay * 4),
LastEvaluationTime: t1, LastEvaluationTime: t1,
@ -458,10 +442,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(6), eval.Normal),
newEvaluation(t1, eval.Normal),
newEvaluation(tn(6), eval.Normal),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: tn(6), LastEvaluationTime: tn(6),
@ -485,10 +466,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.Alerting),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Alerting),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -518,10 +496,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(4), eval.Alerting),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.Alerting),
},
StartsAt: tn(4), StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4), EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4), LastEvaluationTime: tn(4),
@ -554,10 +529,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending, State: eval.Pending,
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(5), eval.Alerting),
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Alerting),
},
StartsAt: tn(4), StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4), EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5), LastEvaluationTime: tn(5),
@ -587,10 +559,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData, State: eval.NoData,
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(4), eval.NoData),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.NoData),
},
StartsAt: tn(4), StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4), EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4), LastEvaluationTime: tn(4),
@ -615,10 +584,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending, State: eval.Pending,
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.Alerting),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Alerting),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -642,10 +608,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending, State: eval.Pending,
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.Alerting),
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Alerting),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4), EndsAt: t1.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -670,10 +633,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending, State: eval.Pending,
StateReason: eval.NoData.String(), StateReason: eval.NoData.String(),
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.NoData),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -707,11 +667,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
StateReason: eval.NoData.String(), StateReason: eval.NoData.String(),
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(5), eval.NoData),
newEvaluation(t3, eval.NoData),
newEvaluation(tn(4), eval.NoData),
newEvaluation(tn(5), eval.NoData),
},
StartsAt: tn(5), StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4), EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5), LastEvaluationTime: tn(5),
@ -735,10 +691,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData, State: eval.NoData,
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.NoData),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -762,9 +715,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: newEvaluation(t1, eval.Normal),
newEvaluation(t1, eval.Normal),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t1, LastEvaluationTime: t1,
@ -773,9 +724,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule"], Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(), ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData, State: eval.NoData,
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.NoData),
newEvaluation(t2, eval.NoData),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -800,9 +749,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: newEvaluation(t1, eval.Normal),
newEvaluation(t1, eval.Normal),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t1, LastEvaluationTime: t1,
@ -811,9 +758,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels2"], Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(), ResultFingerprint: labels2.Fingerprint(),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: newEvaluation(t1, eval.Normal),
newEvaluation(t1, eval.Normal),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t1, LastEvaluationTime: t1,
@ -822,9 +767,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule"], Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(), ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData, State: eval.NoData,
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.NoData),
newEvaluation(t2, eval.NoData),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -851,10 +794,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: newEvaluation(t3, eval.Normal),
newEvaluation(t1, eval.Normal),
newEvaluation(t3, eval.Normal),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t3, LastEvaluationTime: t3,
@ -863,9 +803,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + no-data"], Labels: labels["system + rule + no-data"],
ResultFingerprint: noDataLabels.Fingerprint(), ResultFingerprint: noDataLabels.Fingerprint(),
State: eval.NoData, State: eval.NoData,
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.NoData),
newEvaluation(t2, eval.NoData),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -896,12 +834,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast), StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(4), eval.NoData),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.NoData),
},
StartsAt: t3, StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4), EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4), LastEvaluationTime: tn(4),
@ -932,10 +865,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast), StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(4), eval.NoData),
newEvaluation(t3, eval.NoData),
newEvaluation(tn(4), eval.NoData),
},
StartsAt: tn(4), StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4), EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4), LastEvaluationTime: tn(4),
@ -960,10 +890,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
StateReason: eval.NoData.String(), StateReason: eval.NoData.String(),
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.NoData),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -989,10 +916,7 @@ func TestProcessEvalResults(t *testing.T) {
State: eval.Pending, State: eval.Pending,
StateReason: eval.Error.String(), StateReason: eval.Error.String(),
Error: errors.New("with_state_error"), Error: errors.New("with_state_error"),
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.Error),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -1027,11 +951,7 @@ func TestProcessEvalResults(t *testing.T) {
State: eval.Alerting, State: eval.Alerting,
StateReason: eval.Error.String(), StateReason: eval.Error.String(),
Error: errors.New("with_state_error"), Error: errors.New("with_state_error"),
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(5), eval.Error),
newEvaluation(t3, eval.Error),
newEvaluation(tn(4), eval.Error),
newEvaluation(tn(5), eval.Error),
},
StartsAt: tn(5), StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4), EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5), LastEvaluationTime: tn(5),
@ -1067,10 +987,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Error, State: eval.Error,
Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")), Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.Error),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4), EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -1103,12 +1020,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast), StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(4), eval.Error),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.Error),
},
StartsAt: t3, StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4), EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4), LastEvaluationTime: tn(4),
@ -1139,10 +1051,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast), StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(4), eval.Error),
newEvaluation(t3, eval.Error),
newEvaluation(tn(4), eval.Error),
},
StartsAt: tn(4), StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4), EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4), LastEvaluationTime: tn(4),
@ -1167,10 +1076,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
StateReason: eval.Error.String(), StateReason: eval.Error.String(),
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.Error),
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -1195,10 +1101,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal, State: eval.Normal,
StateReason: eval.Error.String(), StateReason: eval.Error.String(),
Results: []state.Evaluation{ LatestResult: newEvaluation(t2, eval.Error),
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
},
StartsAt: t2, StartsAt: t2,
EndsAt: t2, EndsAt: t2,
LastEvaluationTime: t2, LastEvaluationTime: t2,
@ -1235,10 +1138,7 @@ func TestProcessEvalResults(t *testing.T) {
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Error, State: eval.Error,
Error: fmt.Errorf("with_state_error"), Error: fmt.Errorf("with_state_error"),
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(6), eval.Error),
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(6), eval.Error),
},
StartsAt: tn(4), StartsAt: tn(4),
EndsAt: tn(6).Add(state.ResendDelay * 4), EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6), LastEvaluationTime: tn(6),
@ -1268,11 +1168,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending, State: eval.Pending,
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(8), eval.Alerting),
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(8), eval.Alerting),
},
StartsAt: tn(8), StartsAt: tn(8),
EndsAt: tn(8).Add(state.ResendDelay * 4), EndsAt: tn(8).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(8), LastEvaluationTime: tn(8),
@ -1302,11 +1198,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"], Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(), ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData, State: eval.NoData,
Results: []state.Evaluation{ LatestResult: newEvaluation(tn(6), eval.NoData),
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(6), eval.NoData),
},
StartsAt: tn(6), StartsAt: tn(6),
EndsAt: tn(6).Add(state.ResendDelay * 4), EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6), LastEvaluationTime: tn(6),
@ -1338,9 +1230,7 @@ func TestProcessEvalResults(t *testing.T) {
"job": "prod/grafana", "job": "prod/grafana",
}), }),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: newEvaluation(t1, eval.Normal),
newEvaluation(t1, eval.Normal),
},
StartsAt: t1, StartsAt: t1,
EndsAt: t1, EndsAt: t1,
LastEvaluationTime: t1, LastEvaluationTime: t1,
@ -1374,11 +1264,7 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule"], Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(), ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.Alerting, State: eval.Alerting,
Results: []state.Evaluation{ LatestResult: newEvaluation(t3, eval.Alerting),
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
newEvaluation(t3, eval.Alerting),
},
StartsAt: t3, StartsAt: t3,
EndsAt: t3.Add(state.ResendDelay * 4), EndsAt: t3.Add(state.ResendDelay * 4),
LastEvaluationTime: t3, LastEvaluationTime: t3,
@ -1627,14 +1513,12 @@ func TestStaleResultsHandler(t *testing.T) {
}, },
Values: make(map[string]float64), Values: make(map[string]float64),
State: eval.Normal, State: eval.Normal,
Results: []state.Evaluation{ LatestResult: &state.Evaluation{
{
EvaluationTime: evaluationTime, EvaluationTime: evaluationTime,
EvaluationState: eval.Normal, EvaluationState: eval.Normal,
Values: make(map[string]*float64), Values: make(map[string]*float64),
Condition: "A", Condition: "A",
}, },
},
StartsAt: evaluationTime, StartsAt: evaluationTime,
EndsAt: evaluationTime, EndsAt: evaluationTime,
LastEvaluationTime: evaluationTime, LastEvaluationTime: evaluationTime,

22
pkg/services/ngalert/state/state.go
View File

@ -38,8 +38,8 @@ type State struct {
// ResultFingerprint is a hash of labels of the result before it is processed by // ResultFingerprint is a hash of labels of the result before it is processed by
ResultFingerprint data.Fingerprint ResultFingerprint data.Fingerprint
// Results contains the result of the current and previous evaluations. // LatestResult contains the result of the most recent evaluation, if available.
Results []Evaluation LatestResult *Evaluation
// Error is set if the current evaluation returned an error. If error is non-nil results // Error is set if the current evaluation returned an error. If error is non-nil results
// can still contain the results of previous evaluations. // can still contain the results of previous evaluations.
@ -427,20 +427,6 @@ func (a *State) Equals(b *State) bool {
data.Labels(a.Annotations).String() == data.Labels(b.Annotations).String() data.Labels(a.Annotations).String() == data.Labels(b.Annotations).String()
} }
func (a *State) TrimResults(alertRule *models.AlertRule) {
numBuckets := int64(alertRule.For.Seconds()) / alertRule.IntervalSeconds
if numBuckets == 0 {
numBuckets = 10 // keep at least 10 evaluations in the event For is set to 0
}
if len(a.Results) < int(numBuckets) {
return
}
newResults := make([]Evaluation, numBuckets)
copy(newResults, a.Results[len(a.Results)-int(numBuckets):])
a.Results = newResults
}
func nextEndsTime(interval int64, evaluatedAt time.Time) time.Time { func nextEndsTime(interval int64, evaluatedAt time.Time) time.Time {
ends := ResendDelay ends := ResendDelay
intv := time.Second * time.Duration(interval) intv := time.Second * time.Duration(interval)
@ -464,11 +450,11 @@ func (a *State) GetLabels(opts ...models.LabelOption) map[string]string {
} }
func (a *State) GetLastEvaluationValuesForCondition() map[string]float64 { func (a *State) GetLastEvaluationValuesForCondition() map[string]float64 {
if len(a.Results) <= 0 { if a.LatestResult == nil {
return nil return nil
} }
lastResult := a.Results[len(a.Results)-1] lastResult := *a.LatestResult
r := make(map[string]float64, len(lastResult.Values)) r := make(map[string]float64, len(lastResult.Values))
for refID, value := range lastResult.Values { for refID, value := range lastResult.Values {

30
pkg/services/ngalert/state/state_test.go
View File

@ -475,9 +475,9 @@ func TestNeedsSending(t *testing.T) {
} }
func TestGetLastEvaluationValuesForCondition(t *testing.T) { func TestGetLastEvaluationValuesForCondition(t *testing.T) {
genState := func(results []Evaluation) *State { genState := func(latestResult *Evaluation) *State {
return &State{ return &State{
Results: results, LatestResult: latestResult,
} }
} }
@ -487,16 +487,7 @@ func TestGetLastEvaluationValuesForCondition(t *testing.T) {
}) })
t.Run("should return value of the condition of the last result", func(t *testing.T) { t.Run("should return value of the condition of the last result", func(t *testing.T) {
expected := rand.Float64() expected := rand.Float64()
evals := []Evaluation{ eval := &Evaluation{
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"A": util.Pointer(rand.Float64()),
},
Condition: "A",
},
{
EvaluationTime: time.Time{}, EvaluationTime: time.Time{},
EvaluationState: 0, EvaluationState: 0,
Values: map[string]*float64{ Values: map[string]*float64{
@ -504,40 +495,35 @@ func TestGetLastEvaluationValuesForCondition(t *testing.T) {
"A": util.Pointer(expected), "A": util.Pointer(expected),
}, },
Condition: "A", Condition: "A",
},
} }
result := genState(evals).GetLastEvaluationValuesForCondition() result := genState(eval).GetLastEvaluationValuesForCondition()
require.Len(t, result, 1) require.Len(t, result, 1)
require.Contains(t, result, "A") require.Contains(t, result, "A")
require.Equal(t, result["A"], expected) require.Equal(t, result["A"], expected)
}) })
t.Run("should return empty map if there is no value for condition", func(t *testing.T) { t.Run("should return empty map if there is no value for condition", func(t *testing.T) {
evals := []Evaluation{ eval := &Evaluation{
{
EvaluationTime: time.Time{}, EvaluationTime: time.Time{},
EvaluationState: 0, EvaluationState: 0,
Values: map[string]*float64{ Values: map[string]*float64{
"C": util.Pointer(rand.Float64()), "C": util.Pointer(rand.Float64()),
}, },
Condition: "A", Condition: "A",
},
} }
result := genState(evals).GetLastEvaluationValuesForCondition() result := genState(eval).GetLastEvaluationValuesForCondition()
require.NotNil(t, result) require.NotNil(t, result)
require.Len(t, result, 0) require.Len(t, result, 0)
}) })
t.Run("should use NaN if value is not defined", func(t *testing.T) { t.Run("should use NaN if value is not defined", func(t *testing.T) {
evals := []Evaluation{ eval := &Evaluation{
{
EvaluationTime: time.Time{}, EvaluationTime: time.Time{},
EvaluationState: 0, EvaluationState: 0,
Values: map[string]*float64{ Values: map[string]*float64{
"A": nil, "A": nil,
}, },
Condition: "A", Condition: "A",
},
} }
result := genState(evals).GetLastEvaluationValuesForCondition() result := genState(eval).GetLastEvaluationValuesForCondition()
require.NotNil(t, result) require.NotNil(t, result)
require.Len(t, result, 1) require.Len(t, result, 1)
require.Contains(t, result, "A") require.Contains(t, result, "A")