mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Add error to annotations on data source errors (#91594)
This commit is contained in:
Alexander Akhmetov
GitHub
parent
5105fb7f3a
commit
c7fdf8ce70
@ -201,6 +201,20 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
"datasource_uid": "1",
|
||||
"ref_id": "A",
|
||||
}
|
||||
genericErrorAnnotations := mergeLabels(
|
||||
baseRule.Annotations,
|
||||
data.Labels{
|
||||
"Error": genericError.Error(),
|
||||
},
|
||||
)
|
||||
datasourceErrorAnnotations := mergeLabels(
|
||||
baseRule.Annotations,
|
||||
data.Labels{
|
||||
"datasource_uid": "datasource_uid_1",
|
||||
"ref_id": "A",
|
||||
"Error": datasourceError.Error(),
|
||||
},
|
||||
)
|
||||
|
||||
labels := map[string]data.Labels{
|
||||
"system + rule": mergeLabels(baseRule.Labels, systemLabels),
|
||||
@ -2780,6 +2794,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
StartsAt: t1,
|
||||
EndsAt: t1.Add(ResendDelay * 4),
|
||||
@ -2798,6 +2813,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t1,
|
||||
@ -2813,6 +2829,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
Labels: labels["system + rule"],
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
@ -2844,9 +2861,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
EndsAt: t1.Add(ResendDelay * 4),
|
||||
LastEvaluationTime: t1,
|
||||
LastSentAt: &t1,
|
||||
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
|
||||
"Error": genericError.Error(),
|
||||
}),
|
||||
Annotations: genericErrorAnnotations,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -2860,6 +2875,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: genericError,
|
||||
Annotations: genericErrorAnnotations,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
StartsAt: t1,
|
||||
EndsAt: t1.Add(ResendDelay * 4),
|
||||
@ -2877,6 +2893,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
Labels: labels["system + rule"],
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
Annotations: genericErrorAnnotations,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
@ -2893,6 +2910,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
Labels: labels["system + rule"],
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
Annotations: genericErrorAnnotations,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
@ -2945,6 +2963,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Pending,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -2962,6 +2981,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t2,
|
||||
EndsAt: t2,
|
||||
LastEvaluationTime: t2,
|
||||
@ -2978,6 +2998,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t2,
|
||||
EndsAt: t2,
|
||||
LastEvaluationTime: t2,
|
||||
@ -2996,6 +3017,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -3014,6 +3036,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
Labels: labels["system + rule + labels1"],
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2,
|
||||
@ -3031,6 +3054,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
Labels: labels["system + rule + labels1"],
|
||||
State: eval.Alerting,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -3084,6 +3108,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -3102,6 +3127,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t2,
|
||||
EndsAt: t2,
|
||||
LastEvaluationTime: t2,
|
||||
@ -3118,6 +3144,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t2,
|
||||
EndsAt: t2,
|
||||
LastEvaluationTime: t2,
|
||||
@ -3136,6 +3163,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -3155,6 +3183,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t2,
|
||||
@ -3172,6 +3201,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
LatestResult: newEvaluationWithValues(t2, eval.Error, map[string]float64{"A": float64(-1)}),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t2,
|
||||
@ -3248,6 +3278,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: &State{
|
||||
Labels: labels["system + rule"],
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.StateReasonMissingSeries,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
@ -3281,6 +3312,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.StateReasonMissingSeries,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t3,
|
||||
LastEvaluationTime: t3,
|
||||
@ -3309,6 +3341,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.StateReasonMissingSeries,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t3,
|
||||
LastEvaluationTime: t3,
|
||||
@ -3359,6 +3392,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -3377,6 +3411,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t2,
|
||||
@ -3393,6 +3428,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t2,
|
||||
@ -3457,6 +3493,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -3474,6 +3511,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
Labels: labels["system + rule"],
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2,
|
||||
@ -3490,6 +3528,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
Labels: labels["system + rule"],
|
||||
State: eval.Alerting,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -3560,6 +3599,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Pending,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t1,
|
||||
EndsAt: t1.Add(ResendDelay * 4), // TODO probably it should be t1 (semantic of Normal)?
|
||||
@ -3592,6 +3632,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t2,
|
||||
EndsAt: t2,
|
||||
LastEvaluationTime: t2,
|
||||
@ -3622,6 +3663,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Pending,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1.Add(ResendDelay * 4),
|
||||
LastEvaluationTime: t2,
|
||||
@ -3770,9 +3812,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
LastEvaluationTime: t2,
|
||||
LastSentAt: &t1,
|
||||
Annotations: mergeLabels(baseRule.Annotations, data.Labels{
|
||||
"Error": genericError.Error(),
|
||||
}),
|
||||
Annotations: genericErrorAnnotations,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -3786,6 +3826,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: datasourceError,
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
StartsAt: t1,
|
||||
EndsAt: t1.Add(ResendDelay * 4),
|
||||
@ -3803,6 +3844,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: genericError,
|
||||
Annotations: genericErrorAnnotations,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t1,
|
||||
EndsAt: t2.Add(ResendDelay * 4),
|
||||
@ -3821,12 +3863,29 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t1,
|
||||
},
|
||||
},
|
||||
},
|
||||
t2: {
|
||||
{
|
||||
PreviousState: eval.Normal,
|
||||
PreviousStateReason: eval.Error.String(),
|
||||
State: &State{
|
||||
Labels: labels["system + rule"],
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: genericErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
ngmodels.KeepLastErrState: {
|
||||
t1: {
|
||||
@ -3837,12 +3896,29 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
LatestResult: newEvaluation(t1, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t1,
|
||||
},
|
||||
},
|
||||
},
|
||||
t2: {
|
||||
{
|
||||
PreviousState: eval.Normal,
|
||||
PreviousStateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
State: &State{
|
||||
Labels: labels["system + rule"],
|
||||
State: eval.Normal,
|
||||
StateReason: ngmodels.ConcatReasons(eval.Error.String(), ngmodels.StateReasonKeepLast),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: genericErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
@ -372,6 +372,8 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
return r
|
||||
}
|
||||
|
||||
datasourceError := expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))
|
||||
|
||||
labels1 := data.Labels{
|
||||
"instance_label": "test-1",
|
||||
}
|
||||
@ -392,6 +394,13 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
"system + rule + no-data": mergeLabels(mergeLabels(noDataLabels, baseRule.Labels), systemLabels),
|
||||
}
|
||||
|
||||
datasourceErrorAnnotations := data.Labels{
|
||||
"annotation": "test",
|
||||
"datasource_uid": "datasource_uid_1",
|
||||
"ref_id": "A",
|
||||
"Error": datasourceError.Error(),
|
||||
}
|
||||
|
||||
// keep it separate to make code folding work correctly.
|
||||
type testCase struct {
|
||||
desc string
|
||||
@ -1042,6 +1051,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
State: eval.Pending,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: errors.New("with_state_error"),
|
||||
Annotations: map[string]string{"annotation": "test", "Error": "with_state_error"},
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(state.ResendDelay * 4),
|
||||
@ -1077,6 +1087,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
State: eval.Alerting,
|
||||
StateReason: eval.Error.String(),
|
||||
Error: errors.New("with_state_error"),
|
||||
Annotations: map[string]string{"annotation": "test", "Error": "with_state_error"},
|
||||
LatestResult: newEvaluation(tn(5), eval.Error),
|
||||
StartsAt: tn(5),
|
||||
EndsAt: tn(5).Add(state.ResendDelay * 4),
|
||||
@ -1093,7 +1104,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
|
||||
},
|
||||
t2: {
|
||||
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
},
|
||||
},
|
||||
expectedAnnotations: 1,
|
||||
@ -1109,7 +1120,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
}),
|
||||
ResultFingerprint: labels1.Fingerprint(),
|
||||
State: eval.Error,
|
||||
Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
|
||||
Error: datasourceError,
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
StartsAt: t2,
|
||||
EndsAt: t2.Add(state.ResendDelay * 4),
|
||||
@ -1128,13 +1139,13 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
|
||||
},
|
||||
t2: {
|
||||
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
},
|
||||
t3: {
|
||||
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
|
||||
},
|
||||
tn(4): {
|
||||
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
},
|
||||
},
|
||||
expectedAnnotations: 1,
|
||||
@ -1149,6 +1160,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: tn(4).Add(state.ResendDelay * 4),
|
||||
LastEvaluationTime: tn(4),
|
||||
LastSentAt: &t3, // Resend delay is 30s, so last sent at is t3.
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -1163,10 +1175,10 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
|
||||
},
|
||||
t3: {
|
||||
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
},
|
||||
tn(4): {
|
||||
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
},
|
||||
},
|
||||
expectedAnnotations: 2,
|
||||
@ -1181,6 +1193,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
EndsAt: tn(4).Add(state.ResendDelay * 4),
|
||||
LastEvaluationTime: tn(4),
|
||||
LastSentAt: util.Pointer(tn(4)),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -1192,7 +1205,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
newResult(eval.WithState(eval.Normal), eval.WithLabels(labels1)),
|
||||
},
|
||||
t2: {
|
||||
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
},
|
||||
},
|
||||
expectedAnnotations: 1,
|
||||
@ -1203,6 +1216,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t1,
|
||||
EndsAt: t1,
|
||||
LastEvaluationTime: t2,
|
||||
@ -1217,7 +1231,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
newResult(eval.WithState(eval.Alerting), eval.WithLabels(labels1)),
|
||||
},
|
||||
t2: {
|
||||
newResult(eval.WithError(expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error"))), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
newResult(eval.WithError(datasourceError), eval.WithLabels(labels1)), // TODO fix it because error labels are different
|
||||
},
|
||||
},
|
||||
expectedAnnotations: 2,
|
||||
@ -1228,6 +1242,7 @@ func TestProcessEvalResults(t *testing.T) {
|
||||
State: eval.Normal,
|
||||
StateReason: eval.Error.String(),
|
||||
LatestResult: newEvaluation(t2, eval.Error),
|
||||
Annotations: datasourceErrorAnnotations,
|
||||
StartsAt: t2,
|
||||
EndsAt: t2,
|
||||
LastEvaluationTime: t2,
|
||||
|
||||
@ -141,25 +141,38 @@ func (a *State) Maintain(interval int64, evaluatedAt time.Time) {
|
||||
a.EndsAt = nextEndsTime(interval, evaluatedAt)
|
||||
}
|
||||
|
||||
// AddErrorAnnotations adds annotations to the state to indicate that an error occurred.
|
||||
func (a *State) AddErrorAnnotations(err error, rule *models.AlertRule) {
|
||||
// AddErrorInformation adds annotations to the state to indicate that an error occurred.
|
||||
// If addDatasourceInfoToLabels is true, the ref_id and datasource_uid are added to the labels,
|
||||
// otherwise, they are added to the annotations.
|
||||
func (a *State) AddErrorInformation(err error, rule *models.AlertRule, addDatasourceInfoToLabels bool) {
|
||||
if err == nil {
|
||||
return
|
||||
}
|
||||
|
||||
a.Annotations["Error"] = err.Error()
|
||||
|
||||
// If the evaluation failed because a query returned an error then add the Ref ID and
|
||||
// Datasource UID as labels
|
||||
// Datasource UID as labels or annotations
|
||||
var utilError errutil.Error
|
||||
if errors.As(a.Error, &utilError) &&
|
||||
(errors.Is(a.Error, expr.QueryError) || errors.Is(a.Error, expr.ConversionError)) {
|
||||
if errors.As(err, &utilError) &&
|
||||
(errors.Is(err, expr.QueryError) || errors.Is(err, expr.ConversionError)) {
|
||||
for _, next := range rule.Data {
|
||||
if next.RefID == utilError.PublicPayload["refId"].(string) {
|
||||
a.Labels["ref_id"] = next.RefID
|
||||
a.Labels["datasource_uid"] = next.DatasourceUID
|
||||
if addDatasourceInfoToLabels {
|
||||
a.Labels["ref_id"] = next.RefID
|
||||
a.Labels["datasource_uid"] = next.DatasourceUID
|
||||
} else {
|
||||
a.Annotations["ref_id"] = next.RefID
|
||||
a.Annotations["datasource_uid"] = next.DatasourceUID
|
||||
}
|
||||
break
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Remove the ref_id and datasource_uid from the annotations if they are present.
|
||||
// It can happen if the alert state hasn't changed, but the error is different now.
|
||||
delete(a.Annotations, "ref_id")
|
||||
delete(a.Annotations, "datasource_uid")
|
||||
}
|
||||
}
|
||||
|
||||
@ -334,11 +347,12 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge
|
||||
resultAlerting(state, rule, result, logger, models.StateReasonError)
|
||||
// This is a special case where Alerting and Pending should also have an error and reason
|
||||
state.Error = result.Error
|
||||
state.AddErrorInformation(result.Error, rule, false)
|
||||
case models.ErrorErrState:
|
||||
if state.State == eval.Error {
|
||||
prevEndsAt := state.EndsAt
|
||||
state.Error = result.Error
|
||||
state.AddErrorAnnotations(result.Error, rule)
|
||||
state.AddErrorInformation(result.Error, rule, true)
|
||||
state.Maintain(rule.IntervalSeconds, result.EvaluatedAt)
|
||||
logger.Debug("Keeping state",
|
||||
"state",
|
||||
@ -360,18 +374,20 @@ func resultError(state *State, rule *models.AlertRule, result eval.Result, logge
|
||||
"next_ends_at",
|
||||
nextEndsAt)
|
||||
state.SetError(result.Error, result.EvaluatedAt, nextEndsAt)
|
||||
state.AddErrorAnnotations(result.Error, rule)
|
||||
state.AddErrorInformation(result.Error, rule, true)
|
||||
}
|
||||
case models.OkErrState:
|
||||
logger.Debug("Execution error state is Normal", "handler", "resultNormal", "previous_handler", handlerStr)
|
||||
resultNormal(state, rule, result, logger, "") // TODO: Should we add a reason?
|
||||
state.AddErrorInformation(result.Error, rule, false)
|
||||
case models.KeepLastErrState:
|
||||
logger := logger.New("previous_handler", handlerStr)
|
||||
resultKeepLast(state, rule, result, logger)
|
||||
state.AddErrorInformation(result.Error, rule, false)
|
||||
default:
|
||||
err := fmt.Errorf("unsupported execution error state: %s", rule.ExecErrState)
|
||||
state.SetError(err, state.StartsAt, nextEndsTime(rule.IntervalSeconds, result.EvaluatedAt))
|
||||
state.Annotations["Error"] = err.Error()
|
||||
state.AddErrorInformation(result.Error, rule, false)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user