IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting return evaluation errors to /rules (#33663)

Browse Source 
* Set and return errors produced by evaluation results

* test fixup
This commit is contained in:
David Parrott 2021-05-04 10:08:12 -07:00 committed by GitHub
parent 39099bf3c0
commit b1a8c67689
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 18 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
pkg/services/ngalert/api/api_prometheus.go
View File

@ -100,7 +100,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
alertingRule := apimodels.AlertingRule{
State: "inactive",
Name: rule.Title,
Query: queryStr, // TODO: don't escape <>& etc
Query: queryStr,
Duration: rule.For.Seconds(),
Annotations: rule.Annotations,
}
@ -144,6 +144,11 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
case eval.NoData:
newRule.Health = "nodata"
}
if alertState.Error != nil {
newRule.LastError = alertState.Error.Error()
newRule.Health = "error"
}
alertingRule.Alerts = append(alertingRule.Alerts, alert)
}

21
pkg/services/ngalert/state/state.go
View File

@ -20,6 +20,7 @@ type State struct {
LastEvaluationTime time.Time
EvaluationDuration time.Duration
Annotations map[string]string
Error error
}
type Evaluation struct {
@ -31,19 +32,13 @@ func resultNormal(alertState *State, result eval.Result) *State {
newState := alertState
if alertState.State != eval.Normal {
newState.EndsAt = result.EvaluatedAt
newState.StartsAt = result.EvaluatedAt
}
newState.Error = result.Error // should be nil since state is not error
newState.State = eval.Normal
return newState
}
// addAnnotation adds an annotation to the state.
func (a *State) addAnnotation(key, value string) {
if a.Annotations == nil {
a.Annotations = make(map[string]string)
}
a.Annotations[key] = value
}
func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result) *State {
switch a.State {
case eval.Alerting:
@ -59,19 +54,16 @@ func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result
a.State = eval.Alerting
a.StartsAt = result.EvaluatedAt
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
a.addAnnotation("alerting_at", result.EvaluatedAt.String())
}
default:
a.StartsAt = result.EvaluatedAt
if !(alertRule.For > 0) {
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
a.State = eval.Alerting
a.addAnnotation("alerting_at", result.EvaluatedAt.String())
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For {
a.State = eval.Alerting
a.addAnnotation("alerting_at", result.EvaluatedAt.String())
} else {
a.State = eval.Pending
}
@ -81,6 +73,7 @@ func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result
}
func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) *State {
a.Error = result.Error
if a.StartsAt.IsZero() {
a.StartsAt = result.EvaluatedAt
}
@ -89,9 +82,6 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) *
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
}
if a.State != eval.Error {
a.addAnnotation("last_error", result.EvaluatedAt.String())
}
switch alertRule.ExecErrState {
case ngModels.AlertingErrState:
@ -110,9 +100,6 @@ func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result)
} else {
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
}
if a.State != eval.NoData {
a.addAnnotation("no_data", result.EvaluatedAt.String())
}
switch alertRule.NoDataState {
case ngModels.Alerting:

16
pkg/services/ngalert/tests/manager_test.go
View File

@ -268,7 +268,7 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: evaluationTime.Add(1 * time.Minute).Add(time.Duration(20) * time.Second),
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "alerting_at": "2021-03-25 00:01:00 +0000 UTC"},
Annotations: map[string]string{"annotation": "test"},
},
},
},
@ -341,7 +341,7 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: evaluationTime.Add(80 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(80 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "alerting_at": "2021-03-25 00:01:20 +0000 UTC"},
Annotations: map[string]string{"annotation": "test"},
},
},
},
@ -463,7 +463,7 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
Annotations: map[string]string{"annotation": "test"},
},
},
},
@ -524,7 +524,7 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
Annotations: map[string]string{"annotation": "test"},
},
},
},
@ -585,7 +585,7 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
Annotations: map[string]string{"annotation": "test"},
},
},
},
@ -647,7 +647,7 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
Annotations: map[string]string{"annotation": "test"},
},
},
},
@ -709,7 +709,7 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
Annotations: map[string]string{"annotation": "test"},
},
},
},
@ -771,7 +771,7 @@ func TestProcessEvalResults(t *testing.T) {
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
Annotations: map[string]string{"annotation": "test"},
},
},
},