mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting return evaluation errors to /rules (#33663)
* Set and return errors produced by evaluation results * test fixup
This commit is contained in:
parent
39099bf3c0
commit
b1a8c67689
@ -100,7 +100,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
|
||||
alertingRule := apimodels.AlertingRule{
|
||||
State: "inactive",
|
||||
Name: rule.Title,
|
||||
Query: queryStr, // TODO: don't escape <>& etc
|
||||
Query: queryStr,
|
||||
Duration: rule.For.Seconds(),
|
||||
Annotations: rule.Annotations,
|
||||
}
|
||||
@ -144,6 +144,11 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
|
||||
case eval.NoData:
|
||||
newRule.Health = "nodata"
|
||||
}
|
||||
|
||||
if alertState.Error != nil {
|
||||
newRule.LastError = alertState.Error.Error()
|
||||
newRule.Health = "error"
|
||||
}
|
||||
alertingRule.Alerts = append(alertingRule.Alerts, alert)
|
||||
}
|
||||
|
||||
|
@ -20,6 +20,7 @@ type State struct {
|
||||
LastEvaluationTime time.Time
|
||||
EvaluationDuration time.Duration
|
||||
Annotations map[string]string
|
||||
Error error
|
||||
}
|
||||
|
||||
type Evaluation struct {
|
||||
@ -31,19 +32,13 @@ func resultNormal(alertState *State, result eval.Result) *State {
|
||||
newState := alertState
|
||||
if alertState.State != eval.Normal {
|
||||
newState.EndsAt = result.EvaluatedAt
|
||||
newState.StartsAt = result.EvaluatedAt
|
||||
}
|
||||
newState.Error = result.Error // should be nil since state is not error
|
||||
newState.State = eval.Normal
|
||||
return newState
|
||||
}
|
||||
|
||||
// addAnnotation adds an annotation to the state.
|
||||
func (a *State) addAnnotation(key, value string) {
|
||||
if a.Annotations == nil {
|
||||
a.Annotations = make(map[string]string)
|
||||
}
|
||||
a.Annotations[key] = value
|
||||
}
|
||||
|
||||
func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result) *State {
|
||||
switch a.State {
|
||||
case eval.Alerting:
|
||||
@ -59,19 +54,16 @@ func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result
|
||||
a.State = eval.Alerting
|
||||
a.StartsAt = result.EvaluatedAt
|
||||
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
|
||||
a.addAnnotation("alerting_at", result.EvaluatedAt.String())
|
||||
}
|
||||
default:
|
||||
a.StartsAt = result.EvaluatedAt
|
||||
if !(alertRule.For > 0) {
|
||||
a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
|
||||
a.State = eval.Alerting
|
||||
a.addAnnotation("alerting_at", result.EvaluatedAt.String())
|
||||
} else {
|
||||
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
|
||||
if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For {
|
||||
a.State = eval.Alerting
|
||||
a.addAnnotation("alerting_at", result.EvaluatedAt.String())
|
||||
} else {
|
||||
a.State = eval.Pending
|
||||
}
|
||||
@ -81,6 +73,7 @@ func (a *State) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result
|
||||
}
|
||||
|
||||
func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) *State {
|
||||
a.Error = result.Error
|
||||
if a.StartsAt.IsZero() {
|
||||
a.StartsAt = result.EvaluatedAt
|
||||
}
|
||||
@ -89,9 +82,6 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) *
|
||||
} else {
|
||||
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
|
||||
}
|
||||
if a.State != eval.Error {
|
||||
a.addAnnotation("last_error", result.EvaluatedAt.String())
|
||||
}
|
||||
|
||||
switch alertRule.ExecErrState {
|
||||
case ngModels.AlertingErrState:
|
||||
@ -110,9 +100,6 @@ func (a *State) resultNoData(alertRule *ngModels.AlertRule, result eval.Result)
|
||||
} else {
|
||||
a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
|
||||
}
|
||||
if a.State != eval.NoData {
|
||||
a.addAnnotation("no_data", result.EvaluatedAt.String())
|
||||
}
|
||||
|
||||
switch alertRule.NoDataState {
|
||||
case ngModels.Alerting:
|
||||
|
@ -268,7 +268,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: evaluationTime.Add(1 * time.Minute).Add(time.Duration(20) * time.Second),
|
||||
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test", "alerting_at": "2021-03-25 00:01:00 +0000 UTC"},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -341,7 +341,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: evaluationTime.Add(80 * time.Second).Add(1 * time.Minute),
|
||||
LastEvaluationTime: evaluationTime.Add(80 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test", "alerting_at": "2021-03-25 00:01:20 +0000 UTC"},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -463,7 +463,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
|
||||
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -524,7 +524,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
|
||||
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -585,7 +585,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
|
||||
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -647,7 +647,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
|
||||
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -709,7 +709,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
|
||||
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -771,7 +771,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
|
||||
LastEvaluationTime: evaluationTime.Add(10 * time.Second),
|
||||
EvaluationDuration: evaluationDuration,
|
||||
Annotations: map[string]string{"annotation": "test", "no_data": "2021-03-25 00:00:10 +0000 UTC"},
|
||||
Annotations: map[string]string{"annotation": "test"},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
Loading…
Reference in New Issue
Block a user