mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Support OK option for Error state (#47670)
* support OK state for Error
This commit is contained in:
parent
17bd741fc0
commit
884c885289
@ -46,4 +46,5 @@ Scopes must have an order to ensure consistency and ease of search, this helps u
|
||||
## Grafana Alerting - main / unreleased
|
||||
|
||||
- [CHANGE] Prometheus Compatible API: Use float-like values for `api/prometheus/grafana/api/v1/alerts` and `api/prometheus/grafana/api/v1/rules` instead of the evaluation string #47216
|
||||
- [BUGFIX] Scheduler: Fix state manager to support OK option of `AlertRule.ExecErrState` #47670
|
||||
- [ENHANCEMENT] Templates: Enable the use of classic condition values in templates #46971
|
@ -1397,6 +1397,152 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "normal -> normal when result is Error and ExecErrState is OK",
|
||||
alertRule: &models.AlertRule{
|
||||
OrgID: 1,
|
||||
Title: "test_title",
|
||||
UID: "test_alert_rule_uid_2",
|
||||
NamespaceUID: "test_namespace_uid",
|
||||
Data: []models.AlertQuery{{
|
||||
RefID: "A",
|
||||
DatasourceUID: "datasource_uid_1",
|
||||
}},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
Labels: map[string]string{"label": "test"},
|
||||
IntervalSeconds: 10,
|
||||
For: 1 * time.Minute,
|
||||
ExecErrState: models.OkErrState,
|
||||
},
|
||||
evalResults: []eval.Results{
|
||||
{
|
||||
eval.Result{
|
||||
Instance: data.Labels{"instance_label": "test"},
|
||||
State: eval.Normal,
|
||||
EvaluatedAt: evaluationTime,
|
||||
EvaluationDuration: evaluationDuration,
|
||||
},
|
||||
},
|
||||
{
|
||||
eval.Result{
|
||||
Instance: data.Labels{"instance_label": "test"},
|
||||
Error: expr.QueryError{
|
||||
RefID: "A",
|
||||
Err: errors.New("this is an error"),
|
||||
},
|
||||
State: eval.Error,
|
||||
EvaluatedAt: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedAnnotations: 0,
|
||||
expectedStates: map[string]*state.State{
|
||||
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
|
||||
AlertRuleUID: "test_alert_rule_uid_2",
|
||||
OrgID: 1,
|
||||
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
|
||||
Labels: data.Labels{
|
||||
"__alert_rule_namespace_uid__": "test_namespace_uid",
|
||||
"__alert_rule_uid__": "test_alert_rule_uid_2",
|
||||
"alertname": "test_title",
|
||||
"label": "test",
|
||||
"instance_label": "test",
|
||||
},
|
||||
State: eval.Normal,
|
||||
Error: nil,
|
||||
Results: []state.Evaluation{
|
||||
{
|
||||
EvaluationTime: evaluationTime,
|
||||
EvaluationState: eval.Normal,
|
||||
Values: make(map[string]*float64),
|
||||
},
|
||||
{
|
||||
EvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationState: eval.Error,
|
||||
Values: make(map[string]*float64),
|
||||
},
|
||||
},
|
||||
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "alerting -> normal when result is Error and ExecErrState is OK",
|
||||
alertRule: &models.AlertRule{
|
||||
OrgID: 1,
|
||||
Title: "test_title",
|
||||
UID: "test_alert_rule_uid_2",
|
||||
NamespaceUID: "test_namespace_uid",
|
||||
Data: []models.AlertQuery{{
|
||||
RefID: "A",
|
||||
DatasourceUID: "datasource_uid_1",
|
||||
}},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
Labels: map[string]string{"label": "test"},
|
||||
IntervalSeconds: 10,
|
||||
For: 1 * time.Minute,
|
||||
ExecErrState: models.OkErrState,
|
||||
},
|
||||
evalResults: []eval.Results{
|
||||
{
|
||||
eval.Result{
|
||||
Instance: data.Labels{"instance_label": "test"},
|
||||
State: eval.Alerting,
|
||||
EvaluatedAt: evaluationTime,
|
||||
EvaluationDuration: evaluationDuration,
|
||||
},
|
||||
},
|
||||
{
|
||||
eval.Result{
|
||||
Instance: data.Labels{"instance_label": "test"},
|
||||
Error: expr.QueryError{
|
||||
RefID: "A",
|
||||
Err: errors.New("this is an error"),
|
||||
},
|
||||
State: eval.Error,
|
||||
EvaluatedAt: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
},
|
||||
},
|
||||
},
|
||||
expectedAnnotations: 2,
|
||||
expectedStates: map[string]*state.State{
|
||||
`[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`: {
|
||||
AlertRuleUID: "test_alert_rule_uid_2",
|
||||
OrgID: 1,
|
||||
CacheId: `[["__alert_rule_namespace_uid__","test_namespace_uid"],["__alert_rule_uid__","test_alert_rule_uid_2"],["alertname","test_title"],["instance_label","test"],["label","test"]]`,
|
||||
Labels: data.Labels{
|
||||
"__alert_rule_namespace_uid__": "test_namespace_uid",
|
||||
"__alert_rule_uid__": "test_alert_rule_uid_2",
|
||||
"alertname": "test_title",
|
||||
"label": "test",
|
||||
"instance_label": "test",
|
||||
},
|
||||
State: eval.Normal,
|
||||
Error: nil,
|
||||
Results: []state.Evaluation{
|
||||
{
|
||||
EvaluationTime: evaluationTime,
|
||||
EvaluationState: eval.Alerting,
|
||||
Values: make(map[string]*float64),
|
||||
},
|
||||
{
|
||||
EvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationState: eval.Error,
|
||||
Values: make(map[string]*float64),
|
||||
},
|
||||
},
|
||||
StartsAt: evaluationTime.Add(10 * time.Second),
|
||||
EndsAt: evaluationTime.Add(10 * time.Second),
|
||||
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
desc: "normal -> alerting -> error -> alerting - it should clear the error",
|
||||
alertRule: &models.AlertRule{
|
||||
|
@ -2,6 +2,7 @@ package state
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -51,8 +52,7 @@ func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]*float
|
||||
}
|
||||
|
||||
func (a *State) resultNormal(_ *ngModels.AlertRule, result eval.Result) {
|
||||
a.Error = result.Error // should be nil since state is not error
|
||||
|
||||
a.Error = nil // should be nil since state is not error
|
||||
if a.State != eval.Normal {
|
||||
a.EndsAt = result.EvaluatedAt
|
||||
a.StartsAt = result.EvaluatedAt
|
||||
@ -88,9 +88,10 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) {
|
||||
a.Error = result.Error
|
||||
|
||||
execErrState := eval.Error
|
||||
if alertRule.ExecErrState == ngModels.AlertingErrState {
|
||||
switch alertRule.ExecErrState {
|
||||
case ngModels.AlertingErrState:
|
||||
execErrState = eval.Alerting
|
||||
} else if alertRule.ExecErrState == ngModels.ErrorErrState {
|
||||
case ngModels.ErrorErrState:
|
||||
// If the evaluation failed because a query returned an error then
|
||||
// update the state with the Datasource UID as a label and the error
|
||||
// message as an annotation so other code can use this metadata to
|
||||
@ -107,6 +108,11 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) {
|
||||
a.Annotations["Error"] = queryError.Error()
|
||||
}
|
||||
execErrState = eval.Error
|
||||
case ngModels.OkErrState:
|
||||
a.resultNormal(alertRule, result)
|
||||
return
|
||||
default:
|
||||
a.Error = fmt.Errorf("cannot map error to a state because option [%s] is not supported. evaluation error: %w", alertRule.ExecErrState, a.Error)
|
||||
}
|
||||
|
||||
switch a.State {
|
||||
|
Loading…
Reference in New Issue
Block a user