Apply templating on alert notifications on OK state (#47355)

* OK notifications using previous evaluation data

* copy rule.EvalMatches to avoid changes to the underlying array

* test cases added/modified

* delete trailing newline

* fix double newline in go import

* add change to the changelog

* specify that this only affects legacy alerting (changelog)

* use current eval data instead of prev eval data

* create evalMatch just once

* code comments, renamings, getTemplateMatches() function

* changelog and docs updated

docs/sources/alerting/old-alerting/add-notification-template.md

@@ -9,7 +9,9 @@ aliases = ["/docs/grafana/latest/alerting/add-notification-template/"]
 
 You can provide detailed information to alert notification recipients by injecting alert query data into an alert notification. This topic explains how you can use alert query labels in alert notifications.
 
-Labels that exist from the evaluation of the alert query can be used in the alert rule name and in the alert notification message fields. The alert label data is injected into the notification fields when the alert is in the alerting state. When there are multiple unique values for the same label, the values are comma-separated.
+Labels that exist from the evaluation of the alert query can be used in the alert rule name and in the alert notification message fields. The alert label data is injected into the notification fields when the alert is evaluated and the notification is about to be sent. When there are multiple unique values for the same label, the values are comma-separated.
+
+When the alert is in `firing` state, the data used to parse the templates corresponds to the series that are violating the alert condition at the moment the notification is sent. For `resolved` alert notifications, all series are used since there are none that are violating the condition.
 
 This topic explains how you can use alert query labels in alert notifications.
 

pkg/services/alerting/conditions/query.go

@@ -56,7 +56,11 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega
 
 	emptySeriesCount := 0
 	evalMatchCount := 0
+
+	// matches represents all the series that violate the alert condition
 	var matches []*alerting.EvalMatch
+	// allMatches capture all evaluation matches irregardless on whether the condition is met or not
+	var allMatches []*alerting.EvalMatch
 
 	for _, series := range seriesList {
 		reducedValue := c.Reducer.Reduce(series)
@@ -72,14 +76,17 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega
 			})
 		}
 
+		em := alerting.EvalMatch{
+			Metric: series.Name,
+			Value:  reducedValue,
+			Tags:   series.Tags,
+		}
+
+		allMatches = append(allMatches, &em)
+
 		if evalMatch {
 			evalMatchCount++
-
-			matches = append(matches, &alerting.EvalMatch{
-				Metric: series.Name,
-				Value:  reducedValue,
-				Tags:   series.Tags,
-			})
+			matches = append(matches, &em)
 		}
 	}
 
@@ -105,6 +112,7 @@ func (c *QueryCondition) Eval(context *alerting.EvalContext, requestHandler lega
 		NoDataFound: emptySeriesCount == len(seriesList),
 		Operator:    c.Operator,
 		EvalMatches: matches,
+		AllMatches:  allMatches,
 	}, nil
 }
 

pkg/services/alerting/eval_context.go

@@ -17,6 +17,7 @@ type EvalContext struct {
 	IsTestRun      bool
 	IsDebug        bool
 	EvalMatches    []*EvalMatch
+	AllMatches     []*EvalMatch
 	Logs           []*ResultLogEntry
 	Error          error
 	ConditionEvals string
@@ -47,6 +48,7 @@ func NewEvalContext(alertCtx context.Context, rule *Rule, requestValidator model
 		Rule:             rule,
 		Logs:             make([]*ResultLogEntry, 0),
 		EvalMatches:      make([]*EvalMatch, 0),
+		AllMatches:       make([]*EvalMatch, 0),
 		Log:              log.New("alerting.evalContext"),
 		PrevAlertState:   rule.State,
 		RequestValidator: requestValidator,
@@ -188,11 +190,13 @@ func getNewStateInternal(c *EvalContext) models.AlertStateType {
 // evaluateNotificationTemplateFields will treat the alert evaluation rule's name and message fields as
 // templates, and evaluate the templates using data from the alert evaluation's tags
 func (c *EvalContext) evaluateNotificationTemplateFields() error {
-	if len(c.EvalMatches) < 1 {
+	matches := c.getTemplateMatches()
+	if len(matches) < 1 {
+		// if there are no series to parse the templates with, return
 		return nil
 	}
 
-	templateDataMap, err := buildTemplateDataMap(c.EvalMatches)
+	templateDataMap, err := buildTemplateDataMap(matches)
 	if err != nil {
 		return err
 	}
@@ -212,6 +216,18 @@ func (c *EvalContext) evaluateNotificationTemplateFields() error {
 	return nil
 }
 
+// getTemplateMatches returns the values we should use to parse the templates
+func (c *EvalContext) getTemplateMatches() []*EvalMatch {
+	// EvalMatches represent series violating the rule threshold,
+	// if we have any, this means the alert is firing and we should use this data to parse the templates.
+	if len(c.EvalMatches) > 0 {
+		return c.EvalMatches
+	}
+
+	// If we don't have any alerting values, use all values to parse the templates.
+	return c.AllMatches
+}
+
 func evaluateTemplate(s string, m map[string]string) (string, error) {
 	for k, v := range m {
 		re, err := regexp.Compile(fmt.Sprintf(`\${%s}`, regexp.QuoteMeta(k)))

pkg/services/alerting/eval_context_test.go

@@ -340,3 +340,66 @@ func TestEvaluateTemplate(t *testing.T) {
 		})
 	}
 }
+
+func TestEvaluateNotificationTemplateFields(t *testing.T) {
+	tests := []struct {
+		name            string
+		evalMatches     []*EvalMatch
+		allMatches      []*EvalMatch
+		expectedName    string
+		expectedMessage string
+	}{
+		{
+			"with evaluation matches",
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value2": "test2"},
+			}},
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value2": "test2"},
+			}},
+			"Rule name: test1",
+			"Rule message: test2",
+		},
+		{
+			"missing key",
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value3": "test2"},
+			}},
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value3": "test2"},
+			}},
+			"Rule name: test1",
+			"Rule message: ${value2}",
+		},
+		{
+			"no evaluation matches, with series",
+			[]*EvalMatch{},
+			[]*EvalMatch{{
+				Tags: map[string]string{"value1": "test1", "value2": "test2"},
+			}},
+			"Rule name: test1",
+			"Rule message: test2",
+		},
+		{
+			"no evaluation matches, no series",
+			[]*EvalMatch{},
+			[]*EvalMatch{},
+			"Rule name: ${value1}",
+			"Rule message: ${value2}",
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(tt *testing.T) {
+			evalContext := NewEvalContext(context.Background(), &Rule{Name: "Rule name: ${value1}", Message: "Rule message: ${value2}", Conditions: []Condition{&conditionStub{firing: true}}}, &validations.OSSPluginRequestValidator{}, nil)
+			evalContext.EvalMatches = test.evalMatches
+			evalContext.AllMatches = test.allMatches
+
+			err := evalContext.evaluateNotificationTemplateFields()
+
+			require.NoError(tt, err)
+			require.Equal(tt, test.expectedName, evalContext.Rule.Name)
+			require.Equal(tt, test.expectedMessage, evalContext.Rule.Message)
+		})
+	}
+}

pkg/services/alerting/eval_handler.go

@@ -68,6 +68,7 @@ func (e *DefaultEvalHandler) Eval(context *EvalContext) {
 		}
 
 		context.EvalMatches = append(context.EvalMatches, cr.EvalMatches...)
+		context.AllMatches = append(context.AllMatches, cr.AllMatches...)
 	}
 
 	context.ConditionEvals = conditionEvals + " = " + strconv.FormatBool(firing)

pkg/services/alerting/interfaces.go

@@ -56,6 +56,7 @@ type ConditionResult struct {
 	NoDataFound bool
 	Operator    string
 	EvalMatches []*EvalMatch
+	AllMatches  []*EvalMatch
 }
 
 // Condition is responsible for evaluating an alert condition.

pkg/services/alerting/notifier_test.go

@@ -25,6 +25,7 @@ func TestNotificationService(t *testing.T) {
 	evalCtx := NewEvalContext(context.Background(), testRule, &validations.OSSPluginRequestValidator{}, store)
 
 	testRuleTemplated := &Rule{Name: "Test latency ${quantile}", Message: "Something is bad on instance ${instance}"}
+
 	evalCtxWithMatch := NewEvalContext(context.Background(), testRuleTemplated, &validations.OSSPluginRequestValidator{}, store)
 	evalCtxWithMatch.EvalMatches = []*EvalMatch{{
 		Tags: map[string]string{

pkg/services/ngalert/CHANGELOG.md

@@ -46,5 +46,6 @@ Scopes must have an order to ensure consistency and ease of search, this helps u
 ## Grafana Alerting - main / unreleased
 
 - [CHANGE] Prometheus Compatible API: Use float-like values for `api/prometheus/grafana/api/v1/alerts` and `api/prometheus/grafana/api/v1/rules` instead of the evaluation string #47216
+- [BUGFIX] (Legacy) Templates: Parse notification templates using all the matches of the alert rule when going from `Alerting` to `OK` in legacy alerting #47355
 - [BUGFIX] Scheduler: Fix state manager to support OK option of `AlertRule.ExecErrState` #47670 
 - [ENHANCEMENT] Templates: Enable the use of classic condition values in templates #46971