Expand the value of math and reduce expressions in annotations and labels (#36611)

* Expand the value of math and reduce expressions in annotations and labels

This commit makes it possible to use the values of reduce and math
expressions in annotations and labels via their RefIDs. It uses the
Stringer interface to ensure that "{{ $values.A }}" still prints the
value in decimal format while also making the labels for each RefID
available with "{{ $values.A.Labels }}" and the float64 value with
"{{ $values.A.Value }}"

pkg/services/ngalert/state/cache.go

@@ -3,6 +3,7 @@ package state
 import (
 	"bytes"
 	"fmt"
+	"strconv"
 	"strings"
 	"sync"
 	text_template "text/template"
@@ -35,12 +36,10 @@ func (c *cache) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result) *
 	c.mtxStates.Lock()
 	defer c.mtxStates.Unlock()
 
-	templateData := make(map[string]string, len(result.Instance)+3)
-	for k, v := range result.Instance {
-		templateData[k] = v
-	}
-	attachRuleLabels(templateData, alertRule)
-	ruleLabels, annotations := c.expandRuleLabelsAndAnnotations(alertRule, templateData)
+	// clone the labels so we don't change eval.Result
+	labels := result.Instance.Copy()
+	attachRuleLabels(labels, alertRule)
+	ruleLabels, annotations := c.expandRuleLabelsAndAnnotations(alertRule, labels, result.Values)
 
 	// if duplicate labels exist, alertRule label will take precedence
 	lbs := mergeLabels(ruleLabels, result.Instance)
@@ -89,11 +88,11 @@ func attachRuleLabels(m map[string]string, alertRule *ngModels.AlertRule) {
 	m[prometheusModel.AlertNameLabel] = alertRule.Title
 }
 
-func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, data map[string]string) (map[string]string, map[string]string) {
+func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, labels map[string]string, values map[string]eval.NumberValueCapture) (map[string]string, map[string]string) {
 	expand := func(original map[string]string) map[string]string {
 		expanded := make(map[string]string, len(original))
 		for k, v := range original {
-			ev, err := expandTemplate(alertRule.Title, v, data)
+			ev, err := expandTemplate(alertRule.Title, v, labels, values)
 			expanded[k] = ev
 			if err != nil {
 				c.log.Error("error in expanding template", "name", k, "value", v, "err", err.Error())
@@ -104,13 +103,28 @@ func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, da
 
 		return expanded
 	}
-
 	return expand(alertRule.Labels), expand(alertRule.Annotations)
 }
 
-func expandTemplate(name, text string, data map[string]string) (result string, resultErr error) {
+// templateCaptureValue represents each value in .Values in the annotations
+// and labels template.
+type templateCaptureValue struct {
+	Labels map[string]string
+	Value  *float64
+}
+
+// String implements the Stringer interface to print the value of each RefID
+// in the template via {{ $values.A }} rather than {{ $values.A.Value }}.
+func (v templateCaptureValue) String() string {
+	if v.Value != nil {
+		return strconv.FormatFloat(*v.Value, 'f', -1, 64)
+	}
+	return "null"
+}
+
+func expandTemplate(name, text string, labels map[string]string, values map[string]eval.NumberValueCapture) (result string, resultErr error) {
 	name = "__alert_" + name
-	text = "{{- $labels := .Labels -}}" + text
+	text = "{{- $labels := .Labels -}}{{- $values := .Values -}}" + text
 	// It'd better to have no alert description than to kill the whole process
 	// if there's a bug in the template.
 	defer func() {
@@ -128,12 +142,22 @@ func expandTemplate(name, text string, data map[string]string) (result string, r
 		return "", fmt.Errorf("error parsing template %v: %s", name, err.Error())
 	}
 	var buffer bytes.Buffer
-	err = tmpl.Execute(&buffer, struct {
+	if err := tmpl.Execute(&buffer, struct {
 		Labels map[string]string
+		Values map[string]templateCaptureValue
 	}{
-		Labels: data,
-	})
-	if err != nil {
+		Labels: labels,
+		Values: func() map[string]templateCaptureValue {
+			m := make(map[string]templateCaptureValue)
+			for k, v := range values {
+				m[k] = templateCaptureValue{
+					Labels: v.Labels,
+					Value:  v.Value,
+				}
+			}
+			return m
+		}(),
+	}); err != nil {
 		return "", fmt.Errorf("error executing template %v: %s", name, err.Error())
 	}
 	return buffer.String(), nil

pkg/services/ngalert/state/cache_test.go

@@ -0,0 +1,34 @@
+package state
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	ptr "github.com/xorcare/pointer"
+)
+
+func TestTemplateCaptureValueStringer(t *testing.T) {
+	cases := []struct {
+		name     string
+		value    templateCaptureValue
+		expected string
+	}{{
+		name:     "nil value returns null",
+		value:    templateCaptureValue{Value: nil},
+		expected: "null",
+	}, {
+		name:     "1.0 is returned as integer value",
+		value:    templateCaptureValue{Value: ptr.Float64(1.0)},
+		expected: "1",
+	}, {
+		name:     "1.1 is returned as decimal value",
+		value:    templateCaptureValue{Value: ptr.Float64(1.1)},
+		expected: "1.1",
+	}}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			assert.Equal(t, c.expected, c.value.String())
+		})
+	}
+}

pkg/services/ngalert/state/manager.go

@@ -155,6 +155,7 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul
 		EvaluationTime:   result.EvaluatedAt,
 		EvaluationState:  result.State,
 		EvaluationString: result.EvaluationString,
+		Values:           NewEvaluationValues(result.Values),
 	})
 	currentState.TrimResults(alertRule)
 	oldState := currentState.State

pkg/services/ngalert/state/manager_test.go

@@ -70,6 +70,7 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					LastEvaluationTime: evaluationTime,
@@ -122,6 +123,7 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					LastEvaluationTime: evaluationTime,
@@ -144,6 +146,7 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Alerting,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime,
@@ -200,10 +203,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(1 * time.Minute),
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
@@ -258,10 +263,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(1 * time.Minute),
 							EvaluationState: eval.Alerting,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime.Add(1 * time.Minute),
@@ -327,14 +334,17 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(10 * time.Second),
 							EvaluationState: eval.Alerting,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(80 * time.Second),
 							EvaluationState: eval.Alerting,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime.Add(80 * time.Second),
@@ -392,10 +402,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(10 * time.Second),
 							EvaluationState: eval.Alerting,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime.Add(10 * time.Second),
@@ -453,10 +465,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Alerting,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(10 * time.Second),
 							EvaluationState: eval.Alerting,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime,
@@ -514,10 +528,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(10 * time.Second),
 							EvaluationState: eval.NoData,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime.Add(10 * time.Second),
@@ -575,10 +591,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(10 * time.Second),
 							EvaluationState: eval.NoData,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime.Add(10 * time.Second),
@@ -636,10 +654,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(10 * time.Second),
 							EvaluationState: eval.NoData,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime.Add(10 * time.Second),
@@ -698,10 +718,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(10 * time.Second),
 							EvaluationState: eval.NoData,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime.Add(10 * time.Second),
@@ -760,10 +782,12 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 						{
 							EvaluationTime:  evaluationTime.Add(10 * time.Second),
 							EvaluationState: eval.Error,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					StartsAt:           evaluationTime.Add(10 * time.Second),
@@ -815,6 +839,7 @@ func TestProcessEvalResults(t *testing.T) {
 						{
 							EvaluationTime:  evaluationTime,
 							EvaluationState: eval.Normal,
+							Values:          make(map[string]state.EvaluationValue),
 						},
 					},
 					LastEvaluationTime: evaluationTime,

pkg/services/ngalert/state/state.go

@@ -28,6 +28,28 @@ type Evaluation struct {
 	EvaluationTime   time.Time
 	EvaluationState  eval.State
 	EvaluationString string
+	// Values contains the RefID and value of reduce and math expressions.
+	// It does not contain values for classic conditions as the values
+	// in classic conditions do not have a RefID.
+	Values map[string]EvaluationValue
+}
+
+// EvaluationValue contains the labels and value for a RefID in an evaluation.
+type EvaluationValue struct {
+	Labels data.Labels
+	Value  *float64
+}
+
+// NewEvaluationValues returns the labels and values for each RefID in the capture.
+func NewEvaluationValues(m map[string]eval.NumberValueCapture) map[string]EvaluationValue {
+	result := make(map[string]EvaluationValue, len(m))
+	for k, v := range m {
+		result[k] = EvaluationValue{
+			Labels: v.Labels,
+			Value:  v.Value,
+		}
+	}
+	return result
 }
 
 func (a *State) resultNormal(alertRule *ngModels.AlertRule, result eval.Result) {