Alerting: Migrate old alerting templates to Go templates (#62911)

* Migrate old alerting templates to use $labels

* Fix imports

* Add test coverage and separate rewriting to Go templates

* Fix lint

* Check for additional closing braces

* Add logging of invalid message templates

* Fix tests

* Small fixes

* Update comments

* Panic on empty token

* Use logtest.Fake

* Fix lint

* Allow for spaces in variable names by not tokenizing spaces

* Add template function to deduplicate Labels in a Value map

* Fix behavior of mapLookupString

* Reference deduplicated labels in migrated message template

* Fix behavior of deduplicateLabelsFunc

* Don't create variable for parent logger

* Add more tests for deduplicateLabelsFunc

* Remove unused function

* Apply suggestions from code review

Co-authored by: Yuri Tseretyan <yuriy.tseretyan@grafana.com>

* Give label val merge function better name

* Extract template migration and escape literal tokens

* Consolidate + simplify template migration

---------

Co-authored-by: William Wernert <william.wernert@grafana.com>

pkg/services/ngalert/state/template/funcs.go

@@ -5,7 +5,10 @@ import (
 	"fmt"
 	"net/url"
 	"regexp"
+	"strings"
 	"text/template"
+
+	"golang.org/x/exp/slices"
 )
 
 type query struct {
@@ -13,14 +16,25 @@ type query struct {
 	Expr       string `json:"expr"`
 }
 
+const (
+	FilterLabelFuncName      = "filterLabels"
+	FilterLabelReFuncName    = "filterLabelsRe"
+	GraphLinkFuncName        = "graphLink"
+	RemoveLabelsFuncName     = "removeLabels"
+	RemoveLabelsReFuncName   = "removeLabelsRe"
+	TableLinkFuncName        = "tableLink"
+	MergeLabelValuesFuncName = "mergeLabelValues"
+)
+
 var (
 	defaultFuncs = template.FuncMap{
-		"filterLabels":    filterLabelsFunc,
-		"filterLabelsRe":  filterLabelsReFunc,
-		"graphLink":       graphLinkFunc,
-		"removeLabels":    removeLabelsFunc,
-		"removeLabelslRe": removeLabelsReFunc,
-		"tableLink":       tableLinkFunc,
+		FilterLabelFuncName:      filterLabelsFunc,
+		FilterLabelReFuncName:    filterLabelsReFunc,
+		GraphLinkFuncName:        graphLinkFunc,
+		RemoveLabelsFuncName:     removeLabelsFunc,
+		RemoveLabelsReFuncName:   removeLabelsReFunc,
+		TableLinkFuncName:        tableLinkFunc,
+		MergeLabelValuesFuncName: mergeLabelValuesFunc,
 	}
 )
 
@@ -89,3 +103,32 @@ func tableLinkFunc(data string) string {
 	expr := url.QueryEscape(q.Expr)
 	return fmt.Sprintf(`/explore?left={"datasource":%[1]q,"queries":[{"datasource":%[1]q,"expr":%q,"instant":true,"range":false,"refId":"A"}],"range":{"from":"now-1h","to":"now"}}`, datasource, expr)
 }
+
+// mergeLabelValuesFunc returns a map of label keys to deduplicated and comma separated values.
+func mergeLabelValuesFunc(values map[string]Value) Labels {
+	type uniqueLabelVals map[string]struct{}
+
+	labels := make(map[string]uniqueLabelVals)
+	for _, value := range values {
+		for k, v := range value.Labels {
+			var ul uniqueLabelVals
+			var ok bool
+			if ul, ok = labels[k]; !ok {
+				ul = uniqueLabelVals{}
+				labels[k] = ul
+			}
+			ul[v] = struct{}{}
+		}
+	}
+
+	res := make(Labels)
+	for label, vals := range labels {
+		keys := make([]string, 0, len(vals))
+		for val := range vals {
+			keys = append(keys, val)
+		}
+		slices.Sort(keys)
+		res[label] = strings.Join(keys, ", ")
+	}
+	return res
+}

pkg/services/ngalert/state/template/funcs_test.go

@@ -25,3 +25,27 @@ func TestRemoveLabelsReFunc(t *testing.T) {
 	l := Labels{"foo": "bar", "bar": "baz"}
 	assert.Equal(t, Labels{"bar": "baz"}, removeLabelsReFunc(l, "f.*"))
 }
+
+func TestDeduplicateLabelsFunc(t *testing.T) {
+	v := map[string]Value{
+		"v1": {Labels: Labels{"foo": "bar", "bar": "foo"}, Value: 1},
+		"v2": {Labels: Labels{"foo": "bar", "bar": "baz", "baz": "bat"}, Value: 2},
+	}
+	assert.Equal(t, Labels{"foo": "bar", "bar": "baz, foo", "baz": "bat"}, mergeLabelValuesFunc(v))
+}
+
+func TestDeduplicateLabelsFuncAllSameVal(t *testing.T) {
+	v := map[string]Value{
+		"v1": {Labels: Labels{"foo": "bar", "bar": "baz"}, Value: 1},
+		"v2": {Labels: Labels{"foo": "bar", "bar": "baz"}, Value: 2},
+	}
+	assert.Equal(t, Labels{"foo": "bar", "bar": "baz"}, mergeLabelValuesFunc(v))
+}
+
+func TestDeduplicateLabelsFuncNoDuplicates(t *testing.T) {
+	v := map[string]Value{
+		"v1": {Labels: Labels{"foo": "bar"}, Value: 1},
+		"v2": {Labels: Labels{"bar": "baz"}, Value: 2},
+	}
+	assert.Equal(t, Labels{"foo": "bar", "bar": "baz"}, mergeLabelValuesFunc(v))
+}

pkg/services/sqlstore/migrations/ualert/alert_rule.go

@@ -108,8 +108,9 @@ func addMigrationInfo(da *dashAlert) (map[string]string, map[string]string) {
 
 func (m *migration) makeAlertRule(l log.Logger, cond condition, da dashAlert, folderUID string) (*alertRule, error) {
 	lbls, annotations := addMigrationInfo(&da)
-	annotations["message"] = da.Message
-	var err error
+
+	message := MigrateTmpl(l.New("field", "message"), da.Message)
+	annotations["message"] = message
 
 	data, err := migrateAlertRuleQueries(l, cond.Data)
 	if err != nil {

pkg/services/sqlstore/migrations/ualert/alert_rule_test.go

@@ -195,6 +195,20 @@ func TestMakeAlertRule(t *testing.T) {
 		require.Nil(t, err)
 		require.Equal(t, string(models.ErrorErrState), ar.ExecErrState)
 	})
+
+	t.Run("migrate message template", func(t *testing.T) {
+		m := newTestMigration(t)
+		da := createTestDashAlert()
+		da.Message = "Instance ${instance} is down"
+		cnd := createTestDashAlertCondition()
+
+		ar, err := m.makeAlertRule(&logtest.Fake{}, cnd, da, "folder")
+		require.Nil(t, err)
+		expected :=
+			"{{- $mergedLabels := mergeLabelValues $values -}}\n" +
+				"Instance {{$mergedLabels.instance}} is down"
+		require.Equal(t, expected, ar.Annotations["message"])
+	})
 }
 
 func createTestDashAlert() dashAlert {

pkg/services/sqlstore/migrations/ualert/template.go

@@ -0,0 +1,208 @@
+// This file contains code that parses templates from old alerting into a sequence
+// of tokens. Each token can be either a string literal or a variable.
+
+package ualert
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"unicode"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/ngalert/state/template"
+)
+
+// Token contains either a string literal or a variable.
+type Token struct {
+	Literal  string
+	Variable string
+}
+
+func (t Token) IsLiteral() bool {
+	return t.Literal != ""
+}
+
+func (t Token) IsVariable() bool {
+	return t.Variable != ""
+}
+
+func (t Token) String() string {
+	if t.IsLiteral() {
+		return t.Literal
+	} else if t.IsVariable() {
+		return t.Variable
+	} else {
+		panic("empty token")
+	}
+}
+
+func MigrateTmpl(l log.Logger, oldTmpl string) string {
+	var newTmpl string
+
+	tokens := tokenizeTmpl(l, oldTmpl)
+	tokens = escapeLiterals(tokens)
+
+	if anyVariableToken(tokens) {
+		tokens = variablesToMapLookups(tokens, "mergedLabels")
+		newTmpl += fmt.Sprintf("{{- $mergedLabels := %s $values -}}\n", template.MergeLabelValuesFuncName)
+	}
+
+	newTmpl += tokensToTmpl(tokens)
+	return newTmpl
+}
+
+func tokenizeTmpl(logger log.Logger, tmpl string) []Token {
+	var (
+		tokens []Token
+		l      int
+		r      int
+		err    error
+	)
+
+	in := []rune(tmpl)
+	for r < len(in) {
+		if !startVariable(in[r:]) {
+			r++
+			continue
+		}
+
+		token, offset, tokenErr := tokenizeVariable(in[r:])
+		if tokenErr != nil {
+			err = errors.Join(err, tokenErr)
+			r += offset
+			continue
+		}
+
+		// we've found a variable, so everything from l -> r is the literal before the variable
+		// ex: "foo ${bar}" -> Literal: "foo ", Variable: "bar"
+		if r > l {
+			tokens = append(tokens, Token{Literal: string(in[l:r])})
+		}
+		tokens = append(tokens, token)
+
+		// seek l and r past the variable
+		r += offset
+		l = r
+	}
+
+	// any remaining runes will be a final literal
+	if r > l {
+		tokens = append(tokens, Token{Literal: string(in[l:r])})
+	}
+
+	if err != nil {
+		logger.Warn("Encountered malformed template", "template", tmpl, "err", err)
+	}
+
+	return tokens
+}
+
+func tokenizeVariable(in []rune) (Token, int, error) {
+	var (
+		pos   int
+		r     rune
+		runes []rune
+	)
+
+	if !startVariable(in) {
+		panic("tokenizeVariable called with input that doesn't start with delimiter")
+	}
+	pos += 2 // seek past opening delimiter
+
+	// consume valid runes until we hit a closing brace
+	// non-space whitespace and the opening delimiter are invalid
+	for pos < len(in) {
+		r = in[pos]
+
+		if unicode.IsSpace(r) && r != ' ' {
+			return Token{}, pos, fmt.Errorf("unexpected whitespace")
+		}
+
+		if startVariable(in[pos:]) {
+			return Token{}, pos, fmt.Errorf("ambiguous delimiter")
+		}
+
+		if r == '}' {
+			pos++
+			break
+		}
+
+		runes = append(runes, r)
+		pos++
+	}
+
+	// variable must end with '}' delimiter
+	if r != '}' {
+		return Token{}, pos, fmt.Errorf("expected '}', got '%c'", r)
+	}
+
+	return Token{Variable: string(runes)}, pos, nil
+}
+
+func startVariable(in []rune) bool {
+	return len(in) >= 2 && in[0] == '$' && in[1] == '{'
+}
+
+func anyVariableToken(tokens []Token) bool {
+	for _, token := range tokens {
+		if token.IsVariable() {
+			return true
+		}
+	}
+	return false
+}
+
+// tokensToTmpl returns the tokens as a Go template
+func tokensToTmpl(tokens []Token) string {
+	buf := bytes.Buffer{}
+	for _, token := range tokens {
+		if token.IsVariable() {
+			buf.WriteString("{{")
+			buf.WriteString(token.String())
+			buf.WriteString("}}")
+		} else {
+			buf.WriteString(token.String())
+		}
+	}
+	return buf.String()
+}
+
+// escapeLiterals escapes any token literals with substrings that would be interpreted as Go template syntax
+func escapeLiterals(tokens []Token) []Token {
+	result := make([]Token, 0, len(tokens))
+	for _, token := range tokens {
+		if token.IsLiteral() && shouldEscape(token.Literal) {
+			token.Literal = fmt.Sprintf("{{`%s`}}", token.Literal)
+		}
+		result = append(result, token)
+	}
+	return result
+}
+
+func shouldEscape(literal string) bool {
+	return strings.Contains(literal, "{{") || literal[len(literal)-1] == '{'
+}
+
+// variablesToMapLookups converts any variables in a slice of tokens to Go template map lookups
+func variablesToMapLookups(tokens []Token, mapName string) []Token {
+	result := make([]Token, 0, len(tokens))
+	for _, token := range tokens {
+		if token.IsVariable() {
+			token.Variable = mapLookupString(token.Variable, mapName)
+		}
+		result = append(result, token)
+	}
+	return result
+}
+
+func mapLookupString(v string, mapName string) string {
+	for _, r := range v {
+		if !(unicode.IsDigit(r) || unicode.IsLetter(r) || r == '_') {
+			return fmt.Sprintf(`index $%s %s`, mapName, strconv.Quote(v)) // quote v to escape any special characters
+		}
+	}
+	return fmt.Sprintf(`$%s.%s`, mapName, v)
+}

pkg/services/sqlstore/migrations/ualert/template_test.go

@@ -0,0 +1,321 @@
+package ualert
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTokenString(t *testing.T) {
+	t1 := Token{Literal: "this is a literal"}
+	assert.Equal(t, "this is a literal", t1.String())
+	t2 := Token{Variable: "this is a variable"}
+	assert.Equal(t, "this is a variable", t2.String())
+}
+
+func TestTokenizeVariable(t *testing.T) {
+	tests := []struct {
+		name   string
+		text   string
+		token  Token
+		offset int
+		err    string
+	}{{
+		name:   "variable with no trailing text",
+		text:   "${instance}",
+		token:  Token{Variable: "instance"},
+		offset: 11,
+	}, {
+		name:   "variable with trailing text",
+		text:   "${instance} is down",
+		token:  Token{Variable: "instance"},
+		offset: 11,
+	}, {
+		name:   "varaiable with numbers",
+		text:   "${instance1} is down",
+		token:  Token{Variable: "instance1"},
+		offset: 12,
+	}, {
+		name:   "variable with underscores",
+		text:   "${instance_with_underscores} is down",
+		token:  Token{Variable: "instance_with_underscores"},
+		offset: 28,
+	}, {
+		name:   "variable with spaces",
+		text:   "${instance with spaces} is down",
+		token:  Token{Variable: "instance with spaces"},
+		offset: 23,
+	}, {
+		name:   "variable with non-reserved special character",
+		text:   "${@instance1} is down",
+		token:  Token{Variable: "@instance1"},
+		offset: 13,
+	}, {
+		name:   "two variables without spaces",
+		text:   "${variable1}${variable2}",
+		token:  Token{Variable: "variable1"},
+		offset: 12,
+	}, {
+		name:   "variable with two closing braces stops at first brace",
+		text:   "${instance}} is down",
+		token:  Token{Variable: "instance"},
+		offset: 11,
+	}, {
+		name:   "variable with newline",
+		text:   "${instance\n} is down",
+		offset: 10,
+		err:    "unexpected whitespace",
+	}, {
+		name:   "variable with ambiguous delimiter returns error",
+		text:   "${${instance}",
+		offset: 2,
+		err:    "ambiguous delimiter",
+	}, {
+		name:   "variable without closing brace returns error",
+		text:   "${instance is down",
+		offset: 18,
+		err:    "expected '}', got 'n'",
+	}}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			token, offset, err := tokenizeVariable([]rune(test.text))
+			if test.err != "" {
+				assert.EqualError(t, err, test.err)
+			}
+			assert.Equal(t, test.offset, offset)
+			assert.Equal(t, test.token, token)
+		})
+	}
+}
+
+func TestTokenizeTmpl(t *testing.T) {
+	tests := []struct {
+		name   string
+		tmpl   string
+		tokens []Token
+	}{{
+		name:   "simple template can be tokenized",
+		tmpl:   "${instance} is down",
+		tokens: []Token{{Variable: "instance"}, {Literal: " is down"}},
+	}, {
+		name: "complex template can be tokenized",
+		tmpl: "More than ${value} ${status_code} in the last 5 minutes",
+		tokens: []Token{
+			{Literal: "More than "},
+			{Variable: "value"},
+			{Literal: " "},
+			{Variable: "status_code"},
+			{Literal: " in the last 5 minutes"},
+		},
+	}, {
+		name:   "variables without spaces between can be tokenized",
+		tmpl:   "${value}${status_code}",
+		tokens: []Token{{Variable: "value"}, {Variable: "status_code"}},
+	}, {
+		name:   "variables without spaces between then literal can be tokenized",
+		tmpl:   "${value}${status_code} in the last 5 minutes",
+		tokens: []Token{{Variable: "value"}, {Variable: "status_code"}, {Literal: " in the last 5 minutes"}},
+	}, {
+		name: "variables with reserved characters can be tokenized",
+		tmpl: "More than ${$value} ${{status_code} in the last 5 minutes",
+		tokens: []Token{
+			{Literal: "More than "},
+			{Variable: "$value"},
+			{Literal: " "},
+			{Variable: "{status_code"},
+			{Literal: " in the last 5 minutes"},
+		},
+	}, {
+		name:   "ambiguous delimiters are tokenized as literals",
+		tmpl:   "Instance ${instance and ${instance} is down",
+		tokens: []Token{{Literal: "Instance ${instance and "}, {Variable: "instance"}, {Literal: " is down"}},
+	}, {
+		name:   "all '$' runes preceding a variable are included in literal",
+		tmpl:   "Instance $${instance} is down",
+		tokens: []Token{{Literal: "Instance $"}, {Variable: "instance"}, {Literal: " is down"}},
+	}, {
+		name:   "sole '$' rune is included in literal",
+		tmpl:   "Instance $instance and ${instance} is down",
+		tokens: []Token{{Literal: "Instance $instance and "}, {Variable: "instance"}, {Literal: " is down"}},
+	}, {
+		name:   "extra closing brace is included in literal",
+		tmpl:   "Instance ${instance}} and ${instance} is down",
+		tokens: []Token{{Literal: "Instance "}, {Variable: "instance"}, {Literal: "} and "}, {Variable: "instance"}, {Literal: " is down"}},
+	}, {
+		name:   "variable with newline tokenized as literal",
+		tmpl:   "${value}${status_code\n}${value} in the last 5 minutes",
+		tokens: []Token{{Variable: "value"}, {Literal: "${status_code\n}"}, {Variable: "value"}, {Literal: " in the last 5 minutes"}},
+	}, {
+		name:   "extra closing brace between variables is included in literal",
+		tmpl:   "${value}${status_code}}${value} in the last 5 minutes",
+		tokens: []Token{{Variable: "value"}, {Variable: "status_code"}, {Literal: "}"}, {Variable: "value"}, {Literal: " in the last 5 minutes"}},
+	}}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			tokens := tokenizeTmpl(log.NewNopLogger(), test.tmpl)
+			assert.Equal(t, test.tokens, tokens)
+		})
+	}
+}
+
+func TestTokensToTmpl(t *testing.T) {
+	tokens := []Token{{Variable: "instance"}, {Literal: " is down"}}
+	assert.Equal(t, "{{instance}} is down", tokensToTmpl(tokens))
+}
+
+func TestTokensToTmplNewlines(t *testing.T) {
+	tokens := []Token{{Variable: "instance"}, {Literal: " is down\n"}, {Variable: "job"}, {Literal: " is down"}}
+	assert.Equal(t, "{{instance}} is down\n{{job}} is down", tokensToTmpl(tokens))
+}
+
+func TestMapLookupString(t *testing.T) {
+	cases := []struct {
+		name     string
+		input    string
+		expected string
+	}{
+		{
+			name:     "when there are no spaces",
+			input:    "instance",
+			expected: "$labels.instance",
+		},
+		{
+			name:     "when there are spaces",
+			input:    "instance with spaces",
+			expected: `index $labels "instance with spaces"`,
+		},
+		{
+			name:     "when there are quotes",
+			input:    `instance with "quotes"`,
+			expected: `index $labels "instance with \"quotes\""`,
+		},
+		{
+			name:     "when there are backslashes",
+			input:    `instance with \backslashes\`,
+			expected: `index $labels "instance with \\backslashes\\"`,
+		},
+		{
+			name:     "when there are legacy delimiter characters",
+			input:    `instance{ with $delim} characters`,
+			expected: `index $labels "instance{ with $delim} characters"`,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expected, mapLookupString(tc.input, "labels"))
+		})
+	}
+}
+
+func TestVariablesToMapLookups(t *testing.T) {
+	tokens := []Token{{Variable: "instance"}, {Literal: " is down"}}
+	expected := []Token{{Variable: "$labels.instance"}, {Literal: " is down"}}
+	assert.Equal(t, expected, variablesToMapLookups(tokens, "labels"))
+}
+
+func TestVariablesToMapLookupsSpace(t *testing.T) {
+	tokens := []Token{{Variable: "instance with spaces"}, {Literal: " is down"}}
+	expected := []Token{{Variable: "index $labels \"instance with spaces\""}, {Literal: " is down"}}
+	assert.Equal(t, expected, variablesToMapLookups(tokens, "labels"))
+}
+
+func TestEscapeLiterals(t *testing.T) {
+	cases := []struct {
+		name     string
+		input    []Token
+		expected []Token
+	}{
+		{
+			name:     "when there are no literals",
+			input:    []Token{{Variable: "instance"}},
+			expected: []Token{{Variable: "instance"}},
+		},
+		{
+			name:     "literal with double braces: {{",
+			input:    []Token{{Literal: "instance {{"}},
+			expected: []Token{{Literal: "{{`instance {{`}}"}},
+		},
+		{
+			name:     "literal that ends with closing brace: {",
+			input:    []Token{{Literal: "instance {"}},
+			expected: []Token{{Literal: "{{`instance {`}}"}},
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			assert.Equal(t, tc.expected, escapeLiterals(tc.input))
+		})
+	}
+}
+
+func TestMigrateTmpl(t *testing.T) {
+	cases := []struct {
+		name     string
+		input    string
+		expected string
+		vars     bool
+	}{
+		{
+			name:     "template does not contain variables",
+			input:    "instance is down",
+			expected: "instance is down",
+			vars:     false,
+		},
+		{
+			name:     "template contains variable",
+			input:    "${instance} is down",
+			expected: withDeduplicateMap("{{$mergedLabels.instance}} is down"),
+			vars:     true,
+		},
+		{
+			name:     "template contains double braces",
+			input:    "{{CRITICAL}} instance is down",
+			expected: "{{`{{CRITICAL}} instance is down`}}",
+			vars:     false,
+		},
+		{
+			name:     "template contains opening brace before variable",
+			input:    `${${instance} is down`,
+			expected: withDeduplicateMap("{{`${`}}{{$mergedLabels.instance}} is down"),
+			vars:     true,
+		},
+		{
+			name:     "template contains newline",
+			input:    "CRITICAL\n${instance} is down",
+			expected: withDeduplicateMap("CRITICAL\n{{$mergedLabels.instance}} is down"),
+			vars:     true,
+		},
+		{
+			name:     "partial migration, no variables",
+			input:    "${instance is down",
+			expected: "${instance is down",
+		},
+		{
+			name:     "partial migration, with variables",
+			input:    "${instance} is down ${${nestedVar}}",
+			expected: withDeduplicateMap("{{$mergedLabels.instance}}{{` is down ${`}}{{$mergedLabels.nestedVar}}}"),
+			vars:     true,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			tmpl := MigrateTmpl(log.NewNopLogger(), tc.input)
+
+			assert.Equal(t, tc.expected, tmpl)
+		})
+	}
+}
+
+func withDeduplicateMap(input string) string {
+	// hardcode function name to fail tests if it changes
+	funcName := "mergeLabelValues"
+
+	return fmt.Sprintf("{{- $mergedLabels := %s $values -}}\n", funcName) + input
+}