IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Do not store series values from past evaluations in state manager for no reason (#87525)

Browse Source 
Do not store previous execution results on states
This commit is contained in:
Alexander Weaver 2024-05-09 15:51:55 -05:00 committed by GitHub
parent c6a0175c04
commit a6a9ab4008
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 902 additions and 1512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pkg/services/ngalert/api/api_prometheus_test.go
View File

@ -45,9 +45,7 @@ func Test_FormatValues(t *testing.T) {
name: "with no value, it renders the evaluation string",
alertState: &state.State{
LastEvaluationString: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
Results: []state.Evaluation{
{Condition: "A", Values: map[string]*float64{}},
},
LatestResult: &state.Evaluation{Condition: "A", Values: map[string]*float64{}},
},
expected: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
},
@ -55,9 +53,7 @@ func Test_FormatValues(t *testing.T) {
name: "with one value, it renders the single value",
alertState: &state.State{
LastEvaluationString: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
Results: []state.Evaluation{
{Condition: "A", Values: map[string]*float64{"A": &val1}},
},
LatestResult: &state.Evaluation{Condition: "A", Values: map[string]*float64{"A": &val1}},
},
expected: "1.1e+00",
},
@ -65,9 +61,7 @@ func Test_FormatValues(t *testing.T) {
name: "with two values, it renders the value based on their refID and position",
alertState: &state.State{
LastEvaluationString: "[ var='B0' metric='vector(10) + time() % 50' labels={} value=1.1 ], [ var='B1' metric='vector(10) + time() % 50' labels={} value=1.4 ]",
Results: []state.Evaluation{
{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2}},
},
LatestResult: &state.Evaluation{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2}},
},
expected: "B0: 1.1e+00, B1: 1.4e+00",
},
@ -75,9 +69,7 @@ func Test_FormatValues(t *testing.T) {
name: "with a high number of values, it renders the value based on their refID and position using a natural order",
alertState: &state.State{
LastEvaluationString: "[ var='B0' metric='vector(10) + time() % 50' labels={} value=1.1 ], [ var='B1' metric='vector(10) + time() % 50' labels={} value=1.4 ]",
Results: []state.Evaluation{
{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2, "B2": &val1, "B10": &val2, "B11": &val1}},
},
LatestResult: &state.Evaluation{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2, "B2": &val1, "B10": &val2, "B11": &val1}},
},
expected: "B0: 1.1e+00, B10: 1.4e+00, B11: 1.1e+00, B1: 1.4e+00, B2: 1.1e+00",
},
@ -246,12 +238,12 @@ func withAlertingState() forEachState {
return func(s *state.State) *state.State {
s.State = eval.Alerting
value := float64(1.1)
s.Results = append(s.Results, state.Evaluation{
s.LatestResult = &state.Evaluation{
EvaluationState: eval.Alerting,
EvaluationTime: timeNow(),
Values: map[string]*float64{"B": &value},
Condition: "B",
})
}
return s
}
}

15
pkg/services/ngalert/api/testing.go
View File

@ -83,17 +83,10 @@ func (f *fakeAlertInstanceManager) GenerateAlertInstances(orgID int64, alertRule
"instance_label": "test",
},
State: eval.Normal,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
{
EvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
LatestResult: &state.Evaluation{
EvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationDuration: evaluationDuration,

6
pkg/services/ngalert/schedule/alert_rule_test.go
View File

@ -311,13 +311,13 @@ func TestRuleRoutine(t *testing.T) {
require.Len(t, states, 1)
s := states[0]
require.Equal(t, rule.UID, s.AlertRuleUID)
require.Len(t, s.Results, 1)
require.NotNil(t, s.LatestResult)
var expectedStatus = evalState
if evalState == eval.Pending {
expectedStatus = eval.Alerting
}
require.Equal(t, expectedStatus.String(), s.Results[0].EvaluationState.String())
require.Equal(t, expectedTime, s.Results[0].EvaluationTime)
require.Equal(t, expectedStatus.String(), s.LatestResult.EvaluationState.String())
require.Equal(t, expectedTime, s.LatestResult.EvaluationTime)
})
t.Run("it should save alert instances to storage", func(t *testing.T) {
// TODO rewrite when we are able to mock/fake state manager

5
pkg/services/ngalert/state/manager.go
View File

@ -358,14 +358,13 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
currentState.LastEvaluationTime = result.EvaluatedAt
currentState.EvaluationDuration = result.EvaluationDuration
currentState.Results = append(currentState.Results, Evaluation{
currentState.LatestResult = &Evaluation{
EvaluationTime: result.EvaluatedAt,
EvaluationState: result.State,
Values: NewEvaluationValues(result.Values),
Condition: alertRule.Condition,
})
}
currentState.LastEvaluationString = result.EvaluationString
currentState.TrimResults(alertRule)
oldState := currentState.State
oldReason := currentState.StateReason

1802
pkg/services/ngalert/state/manager_private_test.go
View File

File diff suppressed because it is too large Load Diff

482
pkg/services/ngalert/state/manager_test.go
View File

@ -53,26 +53,22 @@ func TestWarmStateCache(t *testing.T) {
expectedEntries := []*state.State{
{
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test1": "testValue1"},
State: eval.Normal,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test1": "testValue1"},
State: eval.Normal,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(math.MaxUint64),
}, {
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test2": "testValue2"},
State: eval.Alerting,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test2": "testValue2"},
State: eval.Alerting,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
@ -80,13 +76,11 @@ func TestWarmStateCache(t *testing.T) {
ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1),
},
{
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test3": "testValue3"},
State: eval.NoData,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.NoData},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test3": "testValue3"},
State: eval.NoData,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.NoData},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
@ -94,13 +88,11 @@ func TestWarmStateCache(t *testing.T) {
ResultFingerprint: data.Fingerprint(0),
},
{
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test4": "testValue4"},
State: eval.Error,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Error},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test4": "testValue4"},
State: eval.Error,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Error},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
@ -108,13 +100,11 @@ func TestWarmStateCache(t *testing.T) {
ResultFingerprint: data.Fingerprint(1),
},
{
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test5": "testValue5"},
State: eval.Pending,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Pending},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test5": "testValue5"},
State: eval.Pending,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Pending},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
@ -226,7 +216,7 @@ func TestWarmStateCache(t *testing.T) {
setCacheID(entry)
cacheEntry := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheID)
if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "Results")); diff != "" {
if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "LatestResult")); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff)
t.FailNow()
}
@ -331,8 +321,8 @@ func TestProcessEvalResults(t *testing.T) {
ExecErrState: models.ErrorErrState,
}
newEvaluation := func(evalTime time.Time, evalState eval.State) state.Evaluation {
return state.Evaluation{
newEvaluation := func(evalTime time.Time, evalState eval.State) *state.Evaluation {
return &state.Evaluation{
EvaluationTime: evalTime,
EvaluationState: evalState,
Values: make(map[string]*float64),
@ -395,12 +385,10 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
@ -419,23 +407,19 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
},
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Alerting,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4),
LastEvaluationTime: t1,
@ -455,13 +439,10 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(tn(6), eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(tn(6), eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: tn(6),
@ -482,13 +463,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -515,13 +493,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
LatestResult: newEvaluation(tn(4), eval.Alerting),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -551,13 +526,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3, // Normal -> Pending, Pending -> NoData, NoData -> Pending
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
LatestResult: newEvaluation(tn(5), eval.Alerting),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
@ -584,13 +556,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -614,11 +583,8 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Alerting),
},
State: eval.Pending,
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -639,13 +605,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -666,14 +629,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -703,15 +663,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t3, eval.NoData),
newEvaluation(tn(4), eval.NoData),
newEvaluation(tn(5), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(tn(5), eval.NoData),
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
@ -732,13 +688,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -759,23 +712,19 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -797,34 +746,28 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -848,24 +791,19 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t3, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t3,
},
{
Labels: labels["system + rule + no-data"],
ResultFingerprint: noDataLabels.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule + no-data"],
ResultFingerprint: noDataLabels.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -892,16 +830,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -928,14 +861,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t3, eval.NoData),
newEvaluation(tn(4), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -956,14 +886,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 0,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
@ -984,15 +911,12 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -1022,16 +946,12 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
Results: []state.Evaluation{
newEvaluation(t3, eval.Error),
newEvaluation(tn(4), eval.Error),
newEvaluation(tn(5), eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
LatestResult: newEvaluation(tn(5), eval.Error),
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
@ -1064,13 +984,10 @@ func TestProcessEvalResults(t *testing.T) {
"datasource_uid": "datasource_uid_1",
"ref_id": "A",
}),
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -1099,16 +1016,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.Error),
StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -1135,14 +1047,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t3, eval.Error),
newEvaluation(tn(4), eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.Error),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -1163,14 +1072,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
@ -1191,14 +1097,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
@ -1231,14 +1134,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: fmt.Errorf("with_state_error"),
Results: []state.Evaluation{
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(6), eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: fmt.Errorf("with_state_error"),
LatestResult: newEvaluation(tn(6), eval.Error),
StartsAt: tn(4),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
@ -1265,14 +1165,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(8), eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
LatestResult: newEvaluation(tn(8), eval.Alerting),
StartsAt: tn(8),
EndsAt: tn(8).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(8),
@ -1299,14 +1195,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(6), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(tn(6), eval.NoData),
StartsAt: tn(6),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
@ -1337,10 +1229,8 @@ func TestProcessEvalResults(t *testing.T) {
"label": "test",
"job": "prod/grafana",
}),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
@ -1371,14 +1261,10 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
newEvaluation(t3, eval.Alerting),
},
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(state.ResendDelay * 4),
LastEvaluationTime: t3,
@ -1627,13 +1513,11 @@ func TestStaleResultsHandler(t *testing.T) {
},
Values: make(map[string]float64),
State: eval.Normal,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
Condition: "A",
},
LatestResult: &state.Evaluation{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
Condition: "A",
},
StartsAt: evaluationTime,
EndsAt: evaluationTime,

22
pkg/services/ngalert/state/state.go
View File

@ -38,8 +38,8 @@ type State struct {
// ResultFingerprint is a hash of labels of the result before it is processed by
ResultFingerprint data.Fingerprint
// Results contains the result of the current and previous evaluations.
Results []Evaluation
// LatestResult contains the result of the most recent evaluation, if available.
LatestResult *Evaluation
// Error is set if the current evaluation returned an error. If error is non-nil results
// can still contain the results of previous evaluations.
@ -427,20 +427,6 @@ func (a *State) Equals(b *State) bool {
data.Labels(a.Annotations).String() == data.Labels(b.Annotations).String()
}
func (a *State) TrimResults(alertRule *models.AlertRule) {
numBuckets := int64(alertRule.For.Seconds()) / alertRule.IntervalSeconds
if numBuckets == 0 {
numBuckets = 10 // keep at least 10 evaluations in the event For is set to 0
}
if len(a.Results) < int(numBuckets) {
return
}
newResults := make([]Evaluation, numBuckets)
copy(newResults, a.Results[len(a.Results)-int(numBuckets):])
a.Results = newResults
}
func nextEndsTime(interval int64, evaluatedAt time.Time) time.Time {
ends := ResendDelay
intv := time.Second * time.Duration(interval)
@ -464,11 +450,11 @@ func (a *State) GetLabels(opts ...models.LabelOption) map[string]string {
}
func (a *State) GetLastEvaluationValuesForCondition() map[string]float64 {
if len(a.Results) <= 0 {
if a.LatestResult == nil {
return nil
}
lastResult := a.Results[len(a.Results)-1]
lastResult := *a.LatestResult
r := make(map[string]float64, len(lastResult.Values))
for refID, value := range lastResult.Values {

62
pkg/services/ngalert/state/state_test.go
View File

@ -475,9 +475,9 @@ func TestNeedsSending(t *testing.T) {
}
func TestGetLastEvaluationValuesForCondition(t *testing.T) {
genState := func(results []Evaluation) *State {
genState := func(latestResult *Evaluation) *State {
return &State{
Results: results,
LatestResult: latestResult,
}
}
@ -487,57 +487,43 @@ func TestGetLastEvaluationValuesForCondition(t *testing.T) {
})
t.Run("should return value of the condition of the last result", func(t *testing.T) {
expected := rand.Float64()
evals := []Evaluation{
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"A": util.Pointer(rand.Float64()),
},
Condition: "A",
},
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"B": util.Pointer(rand.Float64()),
"A": util.Pointer(expected),
},
Condition: "A",
eval := &Evaluation{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"B": util.Pointer(rand.Float64()),
"A": util.Pointer(expected),
},
Condition: "A",
}
result := genState(evals).GetLastEvaluationValuesForCondition()
result := genState(eval).GetLastEvaluationValuesForCondition()
require.Len(t, result, 1)
require.Contains(t, result, "A")
require.Equal(t, result["A"], expected)
})
t.Run("should return empty map if there is no value for condition", func(t *testing.T) {
evals := []Evaluation{
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"C": util.Pointer(rand.Float64()),
},
Condition: "A",
eval := &Evaluation{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"C": util.Pointer(rand.Float64()),
},
Condition: "A",
}
result := genState(evals).GetLastEvaluationValuesForCondition()
result := genState(eval).GetLastEvaluationValuesForCondition()
require.NotNil(t, result)
require.Len(t, result, 0)
})
t.Run("should use NaN if value is not defined", func(t *testing.T) {
evals := []Evaluation{
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"A": nil,
},
Condition: "A",
eval := &Evaluation{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"A": nil,
},
Condition: "A",
}
result := genState(evals).GetLastEvaluationValuesForCondition()
result := genState(eval).GetLastEvaluationValuesForCondition()
require.NotNil(t, result)
require.Len(t, result, 1)
require.Contains(t, result, "A")