IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Support OK option for Error state (#47670)

Browse Source 
* support OK state for Error
This commit is contained in:
Yuriy Tseretyan 2022-04-13 14:45:29 -04:00 committed by GitHub
parent 17bd741fc0
commit 884c885289
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 157 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
pkg/services/ngalert/CHANGELOG.md
View File

@ -46,4 +46,5 @@ Scopes must have an order to ensure consistency and ease of search, this helps u
## Grafana Alerting - main / unreleased
- [CHANGE] Prometheus Compatible API: Use float-like values for `api/prometheus/grafana/api/v1/alerts` and `api/prometheus/grafana/api/v1/rules` instead of the evaluation string #47216
- [BUGFIX] Scheduler: Fix state manager to support OK option of `AlertRule.ExecErrState` #47670
- [ENHANCEMENT] Templates: Enable the use of classic condition values in templates #46971

146
pkg/services/ngalert/state/manager_test.go
View File

@ -1397,6 +1397,152 @@ func TestProcessEvalResults(t *testing.T) {
},
},
},
{
desc: "normal -> normal when result is Error and ExecErrState is OK",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Data: []models.AlertQuery{{
RefID: "A",
DatasourceUID: "datasource_uid_1",
}},
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
ExecErrState: models.OkErrState,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Normal,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
Error: expr.QueryError{
RefID: "A",
Err: errors.New("this is an error"),
},
State: eval.Error,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedAnnotations: 0,
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Error: nil,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.Error,
Values: make(map[string]*float64),
},
},
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "alerting -> normal when result is Error and ExecErrState is OK",
alertRule: &models.AlertRule{
OrgID: 1,
Title: "test_title",
UID: "test_alert_rule_uid_2",
NamespaceUID: "test_namespace_uid",
Data: []models.AlertQuery{{
RefID: "A",
DatasourceUID: "datasource_uid_1",
}},
Annotations: map[string]string{"annotation": "test"},
Labels: map[string]string{"label": "test"},
IntervalSeconds: 10,
For: 1 * time.Minute,
ExecErrState: models.OkErrState,
},
evalResults: []eval.Results{
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
State: eval.Alerting,
EvaluatedAt: evaluationTime,
EvaluationDuration: evaluationDuration,
},
},
{
eval.Result{
Instance: data.Labels{"instance_label": "test"},
Error: expr.QueryError{
RefID: "A",
Err: errors.New("this is an error"),
},
State: eval.Error,
EvaluatedAt: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
},
},
},
expectedAnnotations: 2,
expectedStates: map[string]*state.State{
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
AlertRuleUID: "test_alert_rule_uid_2",
OrgID: 1,
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
Labels: data.Labels{
"__alert_rule_namespace_uid__": "test_namespace_uid",
"__alert_rule_uid__": "test_alert_rule_uid_2",
"alertname": "test_title",
"label": "test",
"instance_label": "test",
},
State: eval.Normal,
Error: nil,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Alerting,
Values: make(map[string]*float64),
},
{
EvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationState: eval.Error,
Values: make(map[string]*float64),
},
},
StartsAt: evaluationTime.Add(10 * time.Second),
EndsAt: evaluationTime.Add(10 * time.Second),
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
EvaluationDuration: evaluationDuration,
Annotations: map[string]string{"annotation": "test"},
},
},
},
{
desc: "normal -> alerting -> error -> alerting - it should clear the error",
alertRule: &models.AlertRule{

14
pkg/services/ngalert/state/state.go
View File

@ -2,6 +2,7 @@ package state
import (
"errors"
"fmt"
"strings"
"time"
@ -51,8 +52,7 @@ func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]*float
}
func (a *State) resultNormal(_ *ngModels.AlertRule, result eval.Result) {
a.Error = result.Error // should be nil since state is not error
a.Error = nil // should be nil since state is not error
if a.State != eval.Normal {
a.EndsAt = result.EvaluatedAt
a.StartsAt = result.EvaluatedAt
@ -88,9 +88,10 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) {
a.Error = result.Error
execErrState := eval.Error
if alertRule.ExecErrState == ngModels.AlertingErrState {
switch alertRule.ExecErrState {
case ngModels.AlertingErrState:
execErrState = eval.Alerting
} else if alertRule.ExecErrState == ngModels.ErrorErrState {
case ngModels.ErrorErrState:
// If the evaluation failed because a query returned an error then
// update the state with the Datasource UID as a label and the error
// message as an annotation so other code can use this metadata to
@ -107,6 +108,11 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) {
a.Annotations["Error"] = queryError.Error()
}
execErrState = eval.Error
case ngModels.OkErrState:
a.resultNormal(alertRule, result)
return
default:
a.Error = fmt.Errorf("cannot map error to a state because option [%s] is not supported. evaluation error: %w", alertRule.ExecErrState, a.Error)
}
switch a.State {