Alerting: Extract values from MD expr alerts (#34757) (#34908)

When using mulit-dimensional Grafana managed alerts (e.g. SSE math) extract refIds values and labels so they can be shown in the notification and dashboards.

(cherry picked from commit b47e7d12e6)

Co-authored-by: Kyle Brandt <kyle@grafana.com>

pkg/services/ngalert/eval/eval.go

@@ -8,6 +8,7 @@ import (
 	"sort"
 	"time"
 
+	"github.com/grafana/grafana/pkg/expr/classic"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 
 	"github.com/grafana/grafana/pkg/setting"
@@ -146,6 +147,12 @@ func GetExprRequest(ctx AlertExecCtx, data []models.AlertQuery, now time.Time) (
 	return req, nil
 }
 
+type NumberValueCapture struct {
+	Var    string // RefID
+	Labels data.Labels
+	Value  *float64
+}
+
 func executeCondition(ctx AlertExecCtx, c *models.Condition, now time.Time, dataService *tsdb.Service) ExecutionResults {
 	result := ExecutionResults{}
 
@@ -155,11 +162,59 @@ func executeCondition(ctx AlertExecCtx, c *models.Condition, now time.Time, data
 		return ExecutionResults{Error: err}
 	}
 
+	// eval captures for the '__value__' label.
+	captures := make([]NumberValueCapture, 0, len(execResp.Responses))
+
+	captureVal := func(refID string, labels data.Labels, value *float64) {
+		captures = append(captures, NumberValueCapture{
+			Var:    refID,
+			Value:  value,
+			Labels: labels.Copy(),
+		})
+	}
+
 	for refID, res := range execResp.Responses {
-		if refID != c.Condition {
-			continue
+		// for each frame within each response, the response can contain several data types including time-series data.
+		// For now, we favour simplicity and only care about single scalar values.
+		for _, frame := range res.Frames {
+			if len(frame.Fields) != 1 || frame.Fields[0].Type() != data.FieldTypeNullableFloat64 {
+				continue
+			}
+			var v *float64
+			if frame.Fields[0].Len() == 1 {
+				v = frame.At(0, 0).(*float64) // type checked above
+			}
+			captureVal(frame.RefID, frame.Fields[0].Labels, v)
+		}
+
+		if refID == c.Condition {
+			result.Results = res.Frames
+		}
+	}
+
+	// add capture values as data frame metadata to each result (frame) that has matching labels.
+	for _, frame := range result.Results {
+		// classic conditions already have metadata set and only have one value, there's no need to add anything in this case.
+		if frame.Meta != nil && frame.Meta.Custom != nil {
+			if _, ok := frame.Meta.Custom.([]classic.EvalMatch); ok {
+				continue // do not overwrite EvalMatch from classic condition.
+			}
+		}
+
+		frame.SetMeta(&data.FrameMeta{}) // overwrite metadata
+
+		if len(frame.Fields) == 1 {
+			theseLabels := frame.Fields[0].Labels
+			for _, cap := range captures {
+				// matching labels are equal labels, or when one set of labels includes the labels of the other.
+				if theseLabels.Equals(cap.Labels) || theseLabels.Contains(cap.Labels) || cap.Labels.Contains(theseLabels) {
+					if frame.Meta.Custom == nil {
+						frame.Meta.Custom = []NumberValueCapture{}
+					}
+					frame.Meta.Custom = append(frame.Meta.Custom.([]NumberValueCapture), cap)
+				}
+			}
 		}
-		result.Results = res.Frames
 	}
 
 	return result

pkg/services/ngalert/eval/extract_md.go

@@ -17,30 +17,51 @@ func extractEvalString(frame *data.Frame) (s string) {
 		return
 	}
 
-	evalMatches, ok := frame.Meta.Custom.([]classic.EvalMatch)
-	if !ok {
-		return
+	if evalMatches, ok := frame.Meta.Custom.([]classic.EvalMatch); ok {
+		sb := strings.Builder{}
+
+		for i, m := range evalMatches {
+			sb.WriteString("[ ")
+			sb.WriteString(fmt.Sprintf("metric='%s' ", m.Metric))
+			sb.WriteString(fmt.Sprintf("labels={%s} ", m.Labels))
+
+			valString := "null"
+			if m.Value != nil {
+				valString = fmt.Sprintf("%v", *m.Value)
+			}
+
+			sb.WriteString(fmt.Sprintf("value=%v ", valString))
+
+			sb.WriteString("]")
+			if i < len(evalMatches)-1 {
+				sb.WriteString(", ")
+			}
+		}
+		return sb.String()
 	}
 
-	sb := strings.Builder{}
+	if caps, ok := frame.Meta.Custom.([]NumberValueCapture); ok {
+		sb := strings.Builder{}
 
-	for i, m := range evalMatches {
-		sb.WriteString("[ ")
-		sb.WriteString(fmt.Sprintf("metric='%s' ", m.Metric))
-		sb.WriteString(fmt.Sprintf("labels={%s} ", m.Labels))
+		for i, c := range caps {
+			sb.WriteString("[ ")
+			sb.WriteString(fmt.Sprintf("var='%s' ", c.Var))
+			sb.WriteString(fmt.Sprintf("labels={%s} ", c.Labels))
 
-		valString := "null"
-		if m.Value != nil {
-			valString = fmt.Sprintf("%v", *m.Value)
-		}
-
-		sb.WriteString(fmt.Sprintf("value=%v ", valString))
-
-		sb.WriteString("]")
-		if i < len(evalMatches)-1 {
-			sb.WriteString(", ")
+			valString := "null"
+			if c.Value != nil {
+				valString = fmt.Sprintf("%v", *c.Value)
+			}
+
+			sb.WriteString(fmt.Sprintf("value=%v ", valString))
+
+			sb.WriteString("]")
+			if i < len(caps)-1 {
+				sb.WriteString(", ")
+			}
 		}
+		return sb.String()
 	}
 
-	return sb.String()
+	return ""
 }

pkg/tests/api/alerting/api_alertmanager_test.go

@@ -1613,7 +1613,7 @@ func TestEval(t *testing.T) {
 					  "Alerting"
 					],
 					[
-					  ""
+					  "[ var='A' labels={} value=1 ]"
 					]
 				  ]
 				}
@@ -1674,7 +1674,7 @@ func TestEval(t *testing.T) {
 					  "Normal"
 					],
 					[
-					  ""
+					  "[ var='A' labels={} value=0 ]"
 					]
 				  ]
 				}

pkg/tests/api/alerting/api_notification_channel_test.go

@@ -1454,7 +1454,7 @@ var expNotifications = map[string][]string{
 			  "startsAt": "%s",
 			  "endsAt": "0001-01-01T00:00:00Z",
 			  "generatorURL": "",
-			  "fingerprint": "929467973978d053",
+			  "fingerprint": "7611eef9e67f6e50",
 			  "silenceURL": "http://localhost:3000/alerting/silence/new?alertmanager=grafana&matchers=alertname%%3DWebhookAlert",
 			  "dashboardURL": "",
 			  "panelURL": ""
@@ -1618,6 +1618,7 @@ var expNotifications = map[string][]string{
 		  {
 			"labels": {
 			  "__alert_rule_uid__": "UID_AlertmanagerAlert",
+        "__value__": "[ var='A' labels={} value=1 ]",
 			  "alertname": "AlertmanagerAlert"
 			},
 			"annotations": {},