Alerting: Set "value" with evalmatches in G Managed (#34075)

When, and currently only when using a classic condition, evaluation information is added (which is like the EvalMatches from dashboard alerting).

This is returned via the API and can be included in notifications by reading the `__value__` label attached `.Alerts` in the template. It is a string.

pkg/services/ngalert/api/api_prometheus.go

@@ -36,12 +36,16 @@ func (srv PrometheusSrv) RouteGetAlertStatuses(c *models.ReqContext) response.Re
 	}
 	for _, alertState := range srv.manager.GetAll(c.OrgId) {
 		startsAt := alertState.StartsAt
+		valString := ""
+		if len(alertState.Results) > 0 && alertState.State == eval.Alerting {
+			valString = alertState.Results[0].EvaluationString
+		}
 		alertResponse.Data.Alerts = append(alertResponse.Data.Alerts, &apimodels.Alert{
 			Labels:      map[string]string(alertState.Labels),
 			Annotations: map[string]string{}, //TODO: Once annotations are added to the evaluation result, set them here
 			State:       alertState.State.String(),
 			ActiveAt:    &startsAt,
-			Value:       "", //TODO: once the result of the evaluation is added to the evaluation result, set it here
+			Value:       valString,
 		})
 	}
 	return response.JSON(http.StatusOK, alertResponse)
@@ -115,12 +119,16 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
 
 			for _, alertState := range srv.manager.GetStatesForRuleUID(c.OrgId, rule.UID) {
 				activeAt := alertState.StartsAt
+				valString := ""
+				if len(alertState.Results) > 0 && alertState.State == eval.Alerting {
+					valString = alertState.Results[0].EvaluationString
+				}
 				alert := &apimodels.Alert{
 					Labels:      map[string]string(alertState.Labels),
 					Annotations: alertState.Annotations,
 					State:       alertState.State.String(),
 					ActiveAt:    &activeAt,
-					Value:       "", // TODO: set this once it is added to the evaluation results
+					Value:       valString, // TODO: set this once it is added to the evaluation results
 				}
 
 				if alertState.LastEvaluationTime.After(newRule.LastEvaluation) {

pkg/services/ngalert/eval/eval.go

@@ -63,6 +63,11 @@ type Result struct {
 	Error              error
 	EvaluatedAt        time.Time
 	EvaluationDuration time.Duration
+
+	// EvaluationSring is a string representation of evaluation data such
+	// as EvalMatches (from "classic condition"), and in the future from operations
+	// like SSE "math".
+	EvaluationString string
 }
 
 // State is an enum of the evaluation State for an alert instance.
@@ -265,6 +270,7 @@ func evaluateExecutionResult(execResults ExecutionResults, ts time.Time) Results
 			Instance:           f.Fields[0].Labels,
 			EvaluatedAt:        ts,
 			EvaluationDuration: time.Since(ts),
+			EvaluationString:   extractEvalString(f),
 		}
 
 		switch {

pkg/services/ngalert/eval/extract_md.go

@@ -0,0 +1,46 @@
+package eval
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/expr/classic"
+)
+
+func extractEvalString(frame *data.Frame) (s string) {
+	if frame == nil {
+		return "empty frame"
+	}
+
+	if frame.Meta == nil || frame.Meta.Custom == nil {
+		return
+	}
+
+	evalMatches, ok := frame.Meta.Custom.([]classic.EvalMatch)
+	if !ok {
+		return
+	}
+
+	sb := strings.Builder{}
+
+	for i, m := range evalMatches {
+		sb.WriteString("[ ")
+		sb.WriteString(fmt.Sprintf("metric='%s' ", m.Metric))
+		sb.WriteString(fmt.Sprintf("labels={%s} ", m.Labels))
+
+		valString := "null"
+		if m.Value != nil {
+			valString = fmt.Sprintf("%v", *m.Value)
+		}
+
+		sb.WriteString(fmt.Sprintf("value=%v ", valString))
+
+		sb.WriteString("]")
+		if i < len(evalMatches)-1 {
+			sb.WriteString(", ")
+		}
+	}
+
+	return sb.String()
+}

pkg/services/ngalert/eval/extract_md_test.go

@@ -0,0 +1,56 @@
+package eval
+
+import (
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/expr/classic"
+	"github.com/stretchr/testify/require"
+	ptr "github.com/xorcare/pointer"
+)
+
+func TestExtractEvalString(t *testing.T) {
+	cases := []struct {
+		desc      string
+		inFrame   *data.Frame
+		outString string
+	}{
+		{
+			desc: "1 EvalMatch",
+			inFrame: newMetaFrame([]classic.EvalMatch{
+				{Metric: "Test", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(32.3)},
+			}, ptr.Float64(1)),
+			outString: `[ metric='Test' labels={host=foo} value=32.3 ]`,
+		},
+		{
+			desc: "2 EvalMatches",
+			inFrame: newMetaFrame([]classic.EvalMatch{
+				{Metric: "Test", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(32.3)},
+				{Metric: "Test", Labels: data.Labels{"host": "baz"}, Value: ptr.Float64(10)},
+			}, ptr.Float64(1)),
+			outString: `[ metric='Test' labels={host=foo} value=32.3 ], [ metric='Test' labels={host=baz} value=10 ]`,
+		},
+		{
+			desc: "3 EvalMatches",
+			inFrame: newMetaFrame([]classic.EvalMatch{
+				{Metric: "Test", Labels: data.Labels{"host": "foo"}, Value: ptr.Float64(32.3)},
+				{Metric: "Test", Labels: data.Labels{"host": "baz"}, Value: ptr.Float64(10)},
+				{Metric: "TestA", Labels: data.Labels{"host": "zip"}, Value: ptr.Float64(11)},
+			}, ptr.Float64(1)),
+			outString: `[ metric='Test' labels={host=foo} value=32.3 ], [ metric='Test' labels={host=baz} value=10 ], [ metric='TestA' labels={host=zip} value=11 ]`,
+		},
+	}
+	for _, tc := range cases {
+		t.Run(tc.desc, func(t *testing.T) {
+			require.Equal(t, tc.outString, extractEvalString(tc.inFrame))
+		})
+	}
+}
+
+func newMetaFrame(custom interface{}, val *float64) *data.Frame {
+	return data.NewFrame("",
+		data.NewField("", nil, []*float64{val})).
+		SetMeta(&data.FrameMeta{
+			Custom: custom,
+		})
+}

pkg/services/ngalert/schedule/compat.go

@@ -14,12 +14,16 @@ func FromAlertStateToPostableAlerts(firingStates []*state.State) apimodels.Posta
 
 	for _, alertState := range firingStates {
 		if alertState.State == eval.Alerting {
+			nL := alertState.Labels.Copy()
+			if len(alertState.Results) > 0 {
+				nL["__value__"] = alertState.Results[0].EvaluationString
+			}
 			alerts.PostableAlerts = append(alerts.PostableAlerts, models.PostableAlert{
 				Annotations: alertState.Annotations,
 				StartsAt:    strfmt.DateTime(alertState.StartsAt),
 				EndsAt:      strfmt.DateTime(alertState.EndsAt),
 				Alert: models.Alert{
-					Labels: models.LabelSet(alertState.Labels),
+					Labels: models.LabelSet(nL),
 				},
 			})
 		}

pkg/services/ngalert/state/manager.go

@@ -74,8 +74,9 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul
 	currentState.LastEvaluationTime = result.EvaluatedAt
 	currentState.EvaluationDuration = result.EvaluationDuration
 	currentState.Results = append(currentState.Results, Evaluation{
-		EvaluationTime:  result.EvaluatedAt,
+		EvaluationTime:   result.EvaluatedAt,
-		EvaluationState: result.State,
+		EvaluationState:  result.State,
+		EvaluationString: result.EvaluationString,
 	})
 
 	st.Log.Debug("setting alert state", "uid", alertRule.UID)

pkg/services/ngalert/state/state.go

@@ -24,8 +24,9 @@ type State struct {
 }
 
 type Evaluation struct {
-	EvaluationTime  time.Time
+	EvaluationTime   time.Time
-	EvaluationState eval.State
+	EvaluationState  eval.State
+	EvaluationString string
 }
 
 func resultNormal(alertState *State, result eval.Result) *State {