From 84e5f336fe1c799924ef544ef5363f47c09dc3dd Mon Sep 17 00:00:00 2001 From: gotjosh Date: Tue, 29 Mar 2022 20:33:03 +0100 Subject: [PATCH] Alerting: Classic conditions can now display multiple values (#46971) * Alerting: Extract classic condition values by RefID * uncapitalise function * update documentation * Update pkg/services/ngalert/eval/extract_md.go Co-authored-by: George Robinson * Update pkg/services/ngalert/state/state.go Co-authored-by: George Robinson * Update pkg/services/ngalert/state/state.go Co-authored-by: George Robinson * Update pkg/services/ngalert/eval/extract_md.go Co-authored-by: George Robinson * Update docs/sources/alerting/unified-alerting/alerting-rules/alert-annotation-label.md Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Update pkg/services/ngalert/eval/extract_md.go Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> * Run prettier Co-authored-by: George Robinson Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com> --- .../alerting-rules/alert-annotation-label.md | 10 ++-- pkg/services/ngalert/eval/extract_md.go | 28 +++++++++-- pkg/services/ngalert/eval/extract_md_test.go | 46 +++++++++++++++---- pkg/services/ngalert/state/state.go | 4 +- 4 files changed, 67 insertions(+), 21 deletions(-) diff --git a/docs/sources/alerting/unified-alerting/alerting-rules/alert-annotation-label.md b/docs/sources/alerting/unified-alerting/alerting-rules/alert-annotation-label.md index 0b88bb1b866..0f421c3df99 100644 --- a/docs/sources/alerting/unified-alerting/alerting-rules/alert-annotation-label.md +++ b/docs/sources/alerting/unified-alerting/alerting-rules/alert-annotation-label.md @@ -31,8 +31,8 @@ Labels are key-value pairs that contain information about, and are used to uniqu The following template variables are available when expanding annotations and labels. -| Name | Description | -| ------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| $labels | The labels from the query or condition. For example, `{{ $labels.instance }}` and `{{ $labels.job }}`. This is unavailable when the rule uses a [classic condition]({{< relref "./create-grafana-managed-rule/#single-and-multi-dimensional-rule" >}}). | -| $values | The values of all reduce and math expressions that were evaluated for this alert rule. For example, `{{ $values.A }}`, `{{ $values.A.Labels }}` and `{{ $values.A.Value }}` where `A` is the `refID` of the expression. This is unavailable when the rule uses a classic condition | -| $value | The value string of the alert instance. For example, `[ var='A' labels={instance=foo} value=10 ]`. | +| Name | Description | +| ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| $labels | The labels from the query or condition. For example, `{{ $labels.instance }}` and `{{ $labels.job }}`. This is unavailable when the rule uses a [classic condition]({{< relref "./create-grafana-managed-rule/#single-and-multi-dimensional-rule" >}}). | +| $values | The values of all reduce and math expressions that were evaluated for this alert rule. For example, `{{ $values.A }}`, `{{ $values.A.Labels }}` and `{{ $values.A.Value }}` where `A` is the `refID` of the expression. If the rule uses classic conditions, then a combination of the `refID` and position of the condition is used. For example, `{{ $values.A0.Value }}` or `{{ $values.A1.Value }}` | +| $value | The value string of the alert instance. For example, `[ var='A' labels={instance=foo} value=10 ]`. | diff --git a/pkg/services/ngalert/eval/extract_md.go b/pkg/services/ngalert/eval/extract_md.go index f908363bab2..8ba7bfb490e 100644 --- a/pkg/services/ngalert/eval/extract_md.go +++ b/pkg/services/ngalert/eval/extract_md.go @@ -20,8 +20,10 @@ func extractEvalString(frame *data.Frame) (s string) { if evalMatches, ok := frame.Meta.Custom.([]classic.EvalMatch); ok { sb := strings.Builder{} + // TODO: Should we simplify when we only have one match and use the name notation of $labels.A? for i, m := range evalMatches { sb.WriteString("[ ") + sb.WriteString(fmt.Sprintf("var='%s%v' ", frame.RefID, i)) sb.WriteString(fmt.Sprintf("metric='%s' ", m.Metric)) sb.WriteString(fmt.Sprintf("labels={%s} ", m.Labels)) @@ -66,10 +68,10 @@ func extractEvalString(frame *data.Frame) (s string) { return "" } -// extractValues returns the RefID and value for all reduce and math expressions -// in the frame. It does not return values for classic conditions as the values -// in classic conditions do not have a RefID. It returns nil if there are -// no results in the frame. +// extractValues returns the RefID and value for all classic conditions, reduce, and math expressions in the frame. +// For classic conditions the same refID can have multiple values due to multiple conditions, for them we use the index of +// the condition in addition to the refID to distinguish between different values. +// It returns nil if there are no results in the frame. func extractValues(frame *data.Frame) map[string]NumberValueCapture { if frame == nil { return nil @@ -77,6 +79,24 @@ func extractValues(frame *data.Frame) map[string]NumberValueCapture { if frame.Meta == nil || frame.Meta.Custom == nil { return nil } + + if matches, ok := frame.Meta.Custom.([]classic.EvalMatch); ok { + // Classic evaluations only have a single match but it can contain multiple conditions. + // Conditions have a strict ordering which we can rely on to distinguish between values. + v := make(map[string]NumberValueCapture, len(matches)) + for i, match := range matches { + // In classic conditions we use refID and the condition position as a way to distinguish between values. + // We can guarantee determinism as conditions are ordered and this order is preserved when marshaling. + refID := fmt.Sprintf("%s%d", frame.RefID, i) + v[refID] = NumberValueCapture{ + Var: frame.RefID, + Labels: match.Labels, + Value: match.Value, + } + } + return v + } + if caps, ok := frame.Meta.Custom.([]NumberValueCapture); ok { v := make(map[string]NumberValueCapture, len(caps)) for _, c := range caps { diff --git a/pkg/services/ngalert/eval/extract_md_test.go b/pkg/services/ngalert/eval/extract_md_test.go index 50a39817f2d..3a1ef7849a9 100644 --- a/pkg/services/ngalert/eval/extract_md_test.go +++ b/pkg/services/ngalert/eval/extract_md_test.go @@ -20,15 +20,15 @@ func TestExtractEvalString(t *testing.T) { inFrame: newMetaFrame([]classic.EvalMatch{ {Metric: "Test", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(32.3)}, }, ptr.Float64(1)), - outString: `[ metric='Test' labels={host=foo} value=32.3 ]`, + outString: `[ var='0' metric='Test' labels={host=foo} value=32.3 ]`, }, { desc: "2 EvalMatches", inFrame: newMetaFrame([]classic.EvalMatch{ {Metric: "Test", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(32.3)}, {Metric: "Test", Labels: data.Labels{"host": "baz"}, Value: ptr.Float64(10)}, - }, ptr.Float64(1)), - outString: `[ metric='Test' labels={host=foo} value=32.3 ], [ metric='Test' labels={host=baz} value=10 ]`, + }, ptr.Float64(1), withRefID("A")), + outString: `[ var='A0' metric='Test' labels={host=foo} value=32.3 ], [ var='A1' metric='Test' labels={host=baz} value=10 ]`, }, { desc: "3 EvalMatches", @@ -36,8 +36,8 @@ func TestExtractEvalString(t *testing.T) { {Metric: "Test", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(32.3)}, {Metric: "Test", Labels: data.Labels{"host": "baz"}, Value: ptr.Float64(10)}, {Metric: "TestA", Labels: data.Labels{"host": "zip"}, Value: ptr.Float64(11)}, - }, ptr.Float64(1)), - outString: `[ metric='Test' labels={host=foo} value=32.3 ], [ metric='Test' labels={host=baz} value=10 ], [ metric='TestA' labels={host=zip} value=11 ]`, + }, ptr.Float64(1), withRefID("A")), + outString: `[ var='A0' metric='Test' labels={host=foo} value=32.3 ], [ var='A1' metric='Test' labels={host=baz} value=10 ], [ var='A2' metric='TestA' labels={host=zip} value=11 ]`, }, } for _, tc := range cases { @@ -57,11 +57,23 @@ func TestExtractValues(t *testing.T) { inFrame: newMetaFrame(nil, ptr.Float64(1)), values: nil, }, { - desc: "Classic condition frame returns nil", + desc: "Classic condition frame with one match", inFrame: newMetaFrame([]classic.EvalMatch{ {Metric: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, - }, ptr.Float64(1)), - values: nil, + }, ptr.Float64(1), withRefID("A")), + values: map[string]NumberValueCapture{ + "A0": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + }, + }, { + desc: "Classic condition frame with multiple matches", + inFrame: newMetaFrame([]classic.EvalMatch{ + {Metric: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + {Metric: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(3)}, + }, ptr.Float64(1), withRefID("A")), + values: map[string]NumberValueCapture{ + "A0": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(1)}, + "A1": {Var: "A", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(3)}, + }, }, { desc: "Nil value", inFrame: newMetaFrame([]NumberValueCapture{ @@ -96,10 +108,24 @@ func TestExtractValues(t *testing.T) { } } -func newMetaFrame(custom interface{}, val *float64) *data.Frame { - return data.NewFrame("", +type frameCallback func(frame *data.Frame) + +func withRefID(refID string) frameCallback { + return func(frame *data.Frame) { + frame.RefID = refID + } +} + +func newMetaFrame(custom interface{}, val *float64, callbacks ...frameCallback) *data.Frame { + f := data.NewFrame("", data.NewField("", nil, []*float64{val})). SetMeta(&data.FrameMeta{ Custom: custom, }) + + for _, cb := range callbacks { + cb(f) + } + + return f } diff --git a/pkg/services/ngalert/state/state.go b/pkg/services/ngalert/state/state.go index 270a029bafa..925e0397556 100644 --- a/pkg/services/ngalert/state/state.go +++ b/pkg/services/ngalert/state/state.go @@ -33,8 +33,8 @@ type Evaluation struct { EvaluationTime time.Time EvaluationState eval.State // Values contains the RefID and value of reduce and math expressions. - // It does not contain values for classic conditions as the values - // in classic conditions do not have a RefID. + // Classic conditions can have different values for the same RefID as they can include multiple conditions. + // For these, we use the index of the condition in addition RefID as the key e.g. "A0, A1, A2, etc.". Values map[string]*float64 }