Alerting template functions (#39261)

* Alerting: (wip) add template funcs

* Alerting: (wip) numeric template functions

* Alerting: (wip) template functions

* Test for the "args" function

* Alerting: (wip) Documentation for template functions

* Alerting: template functions - refactor

* code review changes

* disable linter error

* Use Prometheus implementation of TemplateExpander

* Update docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* change templateCaptureValue to support using template functions

* Update pkg/services/ngalert/state/template.go

Co-authored-by: gotjosh <josue.abreu@gmail.com>

* Test and documentation added for reReplaceAll template function

* complete missing functions, documentation and tests

* Use the alert instance's evaluation time for expanding the template

* strvalue graphlink and tablelink functions

* delete duplicate test

* make strvalue return an empty string

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>
Co-authored-by: gotjosh <josue.abreu@gmail.com>

docs/sources/alerting/unified-alerting/alerting-rules/create-grafana-managed-rule.md

@@ -112,6 +112,36 @@ The following template variables are available when expanding annotations and la
 | $values | The values of all reduce and math expressions that were evaluated for this alert rule. For example, `{{ $values.A }}`, `{{ $values.A.Labels }}` and `{{ $values.A.Value }}` where `A` is the `refID` of the expression. This is unavailable when the rule uses a classic condition. |
 | $value  | The value string of the alert instance. For example, `[ var='A' labels={instance=foo} value=10 ]`.                                                                                                                                                                                  |
 
+#### Template functions
+
+The following template functions are available when expanding annotations and labels.
+
+| Name               | Argument                   | Return                 | Description                                                                                                                        |
+| ------------------ | -------------------------- | ---------------------- | ---------------------------------------------------------------------------------------------------------------------------------- |
+| humanize           | number or string           | string                 | Converts a number to a more readable format, using metric prefixes.                                                                |
+| humanize1024       | number or string           | string                 | Like humanize, but uses 1024 as the base rather than 1000.                                                                         |
+| humanizeDuration   | number or string           | string                 | Converts a duration in seconds to a more readable format.                                                                          |
+| humanizePercentage | number or string           | string                 | Converts a ratio value to a fraction of 100.                                                                                       |
+| humanizeTimestamp  | number or string           | string                 | Converts a Unix timestamp in seconds to a more readable format.                                                                    |
+| title              | string                     | string                 | strings.Title, capitalises first character of each word.                                                                           |
+| toUpper            | string                     | string                 | strings.ToUpper, converts all characters to upper case.                                                                            |
+| toLower            | string                     | string                 | strings.ToLower, converts all characters to lower case.                                                                            |
+| match              | pattern, text              | boolean                | regexp.MatchString Tests for a unanchored regexp match.                                                                            |
+| reReplaceAll       | pattern, replacement, text | string                 | Regexp.ReplaceAllString Regexp substitution, unanchored.                                                                           |
+| graphLink          | expr                       | string                 | Not supported                                                                                                                      |
+| tableLink          | expr                       | string                 | Not supported                                                                                                                      |
+| args               | []interface{}              | map[string]interface{} | Converts a list of objects to a map with keys, for example, arg0, arg1. Use this function to pass multiple arguments to templates. |
+| externalURL        | nothing                    | string                 | Returns a string representing the external URL.                                                                                    |
+| pathPrefix         | nothing                    | string                 | Returns the path of the external URL.                                                                                              |
+| tmpl               | string, []interface{}      | nothing                | Not supported.                                                                                                                     |
+| safeHtml           | string                     | string                 | Not supported.                                                                                                                     |
+| query              | query string               | []sample               | Not supported.                                                                                                                     |
+| first              | []sample                   | sample                 | Not supported.                                                                                                                     |
+| label              | label, sample              | string                 | Not supported.                                                                                                                     |
+| strvalue           | []sample                   | string                 | Not supported.                                                                                                                     |
+| value              | sample                     | float64                | Not supported.                                                                                                                     |
+| sortByLabel        | label, []samples           | []sample               | Not supported.                                                                                                                     |
+
 ## Preview alerts
 
 To evaluate the rule and see what alerts it would produce, click **Preview alerts**. It will display a list of alerts with state and value for each one.

pkg/services/ngalert/ngalert.go

@@ -132,7 +132,7 @@ func (ng *AlertNG) init() error {
 		ng.Log.Error("Failed to parse application URL. Continue without it.", "error", err)
 		appUrl = nil
 	}
-	stateManager := state.NewManager(ng.Log, ng.Metrics.GetStateMetrics(), store, store)
+	stateManager := state.NewManager(ng.Log, ng.Metrics.GetStateMetrics(), appUrl, store, store)
 	scheduler := schedule.NewScheduler(schedCfg, ng.DataService, appUrl, stateManager)
 
 	ng.stateManager = stateManager

pkg/services/ngalert/schedule/schedule_test.go

@@ -104,7 +104,7 @@ func TestWarmStateCache(t *testing.T) {
 		Metrics:                 testMetrics.GetSchedulerMetrics(),
 		AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
 	}
-	st := state.NewManager(schedCfg.Logger, testMetrics.GetStateMetrics(), dbstore, dbstore)
+	st := state.NewManager(schedCfg.Logger, testMetrics.GetStateMetrics(), nil, dbstore, dbstore)
 	st.Warm()
 
 	t.Run("instance cache has expected entries", func(t *testing.T) {
@@ -155,7 +155,7 @@ func TestAlertingTicker(t *testing.T) {
 			disabledOrgID: {},
 		},
 	}
-	st := state.NewManager(schedCfg.Logger, testMetrics.GetStateMetrics(), dbstore, dbstore)
+	st := state.NewManager(schedCfg.Logger, testMetrics.GetStateMetrics(), nil, dbstore, dbstore)
 	appUrl := &url.URL{
 		Scheme: "http",
 		Host:   "localhost",

pkg/services/ngalert/schedule/schedule_unit_test.go

@@ -247,7 +247,7 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac
 		Metrics:                 m.GetSchedulerMetrics(),
 		AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
 	}
-	st := state.NewManager(schedCfg.Logger, m.GetStateMetrics(), rs, is)
+	st := state.NewManager(schedCfg.Logger, m.GetStateMetrics(), nil, rs, is)
 	appUrl := &url.URL{
 		Scheme: "http",
 		Host:   "localhost",

pkg/services/ngalert/state/cache.go

@@ -1,13 +1,10 @@
 package state
 
 import (
-	"bytes"
 	"fmt"
-	"math"
+	"net/url"
-	"strconv"
 	"strings"
 	"sync"
-	text_template "text/template"
 
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 
@@ -19,17 +16,19 @@ import (
 )
 
 type cache struct {
-	states    map[int64]map[string]map[string]*State // orgID > alertRuleUID > stateID > state
+	states      map[int64]map[string]map[string]*State // orgID > alertRuleUID > stateID > state
-	mtxStates sync.RWMutex
+	mtxStates   sync.RWMutex
-	log       log.Logger
+	log         log.Logger
-	metrics   *metrics.State
+	metrics     *metrics.State
+	externalURL *url.URL
 }
 
-func newCache(logger log.Logger, metrics *metrics.State) *cache {
+func newCache(logger log.Logger, metrics *metrics.State, externalURL *url.URL) *cache {
 	return &cache{
-		states:  make(map[int64]map[string]map[string]*State),
+		states:      make(map[int64]map[string]map[string]*State),
-		log:     logger,
+		log:         logger,
-		metrics: metrics,
+		metrics:     metrics,
+		externalURL: externalURL,
 	}
 }
 
@@ -93,7 +92,7 @@ func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, la
 	expand := func(original map[string]string) map[string]string {
 		expanded := make(map[string]string, len(original))
 		for k, v := range original {
-			ev, err := expandTemplate(alertRule.Title, v, labels, alertInstance)
+			ev, err := expandTemplate(alertRule.Title, v, labels, alertInstance, c.externalURL)
 			expanded[k] = ev
 			if err != nil {
 				c.log.Error("error in expanding template", "name", k, "value", v, "err", err.Error())
@@ -107,70 +106,6 @@ func (c *cache) expandRuleLabelsAndAnnotations(alertRule *ngModels.AlertRule, la
 	return expand(alertRule.Labels), expand(alertRule.Annotations)
 }
 
-// templateCaptureValue represents each value in .Values in the annotations
-// and labels template.
-type templateCaptureValue struct {
-	Labels map[string]string
-	Value  float64
-}
-
-// String implements the Stringer interface to print the value of each RefID
-// in the template via {{ $values.A }} rather than {{ $values.A.Value }}.
-func (v templateCaptureValue) String() string {
-	return strconv.FormatFloat(v.Value, 'f', -1, 64)
-}
-
-func expandTemplate(name, text string, labels map[string]string, alertInstance eval.Result) (result string, resultErr error) {
-	name = "__alert_" + name
-	text = "{{- $labels := .Labels -}}{{- $values := .Values -}}{{- $value := .Value -}}" + text
-	// It'd better to have no alert description than to kill the whole process
-	// if there's a bug in the template.
-	defer func() {
-		if r := recover(); r != nil {
-			var ok bool
-			resultErr, ok = r.(error)
-			if !ok {
-				resultErr = fmt.Errorf("panic expanding template %v: %v", name, r)
-			}
-		}
-	}()
-
-	tmpl, err := text_template.New(name).Option("missingkey=error").Parse(text)
-	if err != nil {
-		return "", fmt.Errorf("error parsing template %v: %s", name, err.Error())
-	}
-	var buffer bytes.Buffer
-	if err := tmpl.Execute(&buffer, struct {
-		Labels map[string]string
-		Values map[string]templateCaptureValue
-		Value  string
-	}{
-		Labels: labels,
-		Values: newTemplateCaptureValues(alertInstance.Values),
-		Value:  alertInstance.EvaluationString,
-	}); err != nil {
-		return "", fmt.Errorf("error executing template %v: %s", name, err.Error())
-	}
-	return buffer.String(), nil
-}
-
-func newTemplateCaptureValues(values map[string]eval.NumberValueCapture) map[string]templateCaptureValue {
-	m := make(map[string]templateCaptureValue)
-	for k, v := range values {
-		var f float64
-		if v.Value != nil {
-			f = *v.Value
-		} else {
-			f = math.NaN()
-		}
-		m[k] = templateCaptureValue{
-			Labels: v.Labels,
-			Value:  f,
-		}
-	}
-	return m
-}
-
 func (c *cache) set(entry *State) {
 	c.mtxStates.Lock()
 	defer c.mtxStates.Unlock()

pkg/services/ngalert/state/cache_test.go

@@ -1,126 +0,0 @@
-package state
-
-import (
-	"errors"
-	"testing"
-
-	"github.com/grafana/grafana-plugin-sdk-go/data"
-	"github.com/grafana/grafana/pkg/services/ngalert/eval"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-	ptr "github.com/xorcare/pointer"
-)
-
-func TestTemplateCaptureValueStringer(t *testing.T) {
-	cases := []struct {
-		name     string
-		value    templateCaptureValue
-		expected string
-	}{{
-		name:     "0 is returned as integer value",
-		value:    templateCaptureValue{Value: 0},
-		expected: "0",
-	}, {
-		name:     "1.0 is returned as integer value",
-		value:    templateCaptureValue{Value: 1.0},
-		expected: "1",
-	}, {
-		name:     "1.1 is returned as decimal value",
-		value:    templateCaptureValue{Value: 1.1},
-		expected: "1.1",
-	}}
-
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			assert.Equal(t, c.expected, c.value.String())
-		})
-	}
-}
-
-func TestExpandTemplate(t *testing.T) {
-	cases := []struct {
-		name          string
-		text          string
-		alertInstance eval.Result
-		labels        data.Labels
-		expected      string
-		expectedError error
-	}{{
-		name:     "labels are expanded into $labels",
-		text:     "{{ $labels.instance }} is down",
-		labels:   data.Labels{"instance": "foo"},
-		expected: "foo is down",
-	}, {
-		name:          "missing label in $labels returns error",
-		text:          "{{ $labels.instance }} is down",
-		labels:        data.Labels{},
-		expectedError: errors.New("error executing template __alert_test: template: __alert_test:1:86: executing \"__alert_test\" at <$labels.instance>: map has no entry for key \"instance\""),
-	}, {
-		name: "values are expanded into $values",
-		text: "{{ $values.A.Labels.instance }} has value {{ $values.A }}",
-		alertInstance: eval.Result{
-			Values: map[string]eval.NumberValueCapture{
-				"A": {
-					Var:    "A",
-					Labels: data.Labels{"instance": "foo"},
-					Value:  ptr.Float64(1),
-				},
-			},
-		},
-		expected: "foo has value 1",
-	}, {
-		name: "values can be passed to template functions such as printf",
-		text: "{{ $values.A.Labels.instance }} has value {{ $values.A.Value | printf \"%.1f\" }}",
-		alertInstance: eval.Result{
-			Values: map[string]eval.NumberValueCapture{
-				"A": {
-					Var:    "A",
-					Labels: data.Labels{"instance": "foo"},
-					Value:  ptr.Float64(1.1),
-				},
-			},
-		},
-		expected: "foo has value 1.1",
-	}, {
-		name: "missing label in $values returns error",
-		text: "{{ $values.A.Labels.instance }} has value {{ $values.A }}",
-		alertInstance: eval.Result{
-			Values: map[string]eval.NumberValueCapture{
-				"A": {
-					Var:    "A",
-					Labels: data.Labels{},
-					Value:  ptr.Float64(1),
-				},
-			},
-		},
-		expectedError: errors.New("error executing template __alert_test: template: __alert_test:1:86: executing \"__alert_test\" at <$values.A.Labels.instance>: map has no entry for key \"instance\""),
-	}, {
-		name: "missing value in $values is returned as NaN",
-		text: "{{ $values.A.Labels.instance }} has value {{ $values.A }}",
-		alertInstance: eval.Result{
-			Values: map[string]eval.NumberValueCapture{
-				"A": {
-					Var:    "A",
-					Labels: data.Labels{"instance": "foo"},
-					Value:  nil,
-				},
-			},
-		},
-		expected: "foo has value NaN",
-	}, {
-		name: "assert value string is expanded into $value",
-		text: "{{ $value }}",
-		alertInstance: eval.Result{
-			EvaluationString: "[ var='A' labels={instance=foo} value=10 ]",
-		},
-		expected: "[ var='A' labels={instance=foo} value=10 ]",
-	}}
-
-	for _, c := range cases {
-		t.Run(c.name, func(t *testing.T) {
-			v, err := expandTemplate("test", c.text, c.labels, c.alertInstance)
-			require.Equal(t, c.expectedError, err)
-			require.Equal(t, c.expected, v)
-		})
-	}
-}

pkg/services/ngalert/state/manager.go

@@ -3,6 +3,7 @@ package state
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"strconv"
 	"time"
 
@@ -32,9 +33,9 @@ type Manager struct {
 	instanceStore store.InstanceStore
 }
 
-func NewManager(logger log.Logger, metrics *metrics.State, ruleStore store.RuleStore, instanceStore store.InstanceStore) *Manager {
+func NewManager(logger log.Logger, metrics *metrics.State, externalURL *url.URL, ruleStore store.RuleStore, instanceStore store.InstanceStore) *Manager {
 	manager := &Manager{
-		cache:         newCache(logger, metrics),
+		cache:         newCache(logger, metrics, externalURL),
 		quit:          make(chan struct{}),
 		ResendDelay:   ResendDelay, // TODO: make this configurable
 		log:           logger,

pkg/services/ngalert/state/manager_test.go

@@ -851,7 +851,7 @@ func TestProcessEvalResults(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		st := state.NewManager(log.New("test_state_manager"), testMetrics.GetStateMetrics(), nil, nil)
+		st := state.NewManager(log.New("test_state_manager"), testMetrics.GetStateMetrics(), nil, nil, nil)
 		t.Run(tc.desc, func(t *testing.T) {
 			for _, res := range tc.evalResults {
 				_ = st.ProcessEvalResults(tc.alertRule, res)
@@ -947,7 +947,7 @@ func TestStaleResultsHandler(t *testing.T) {
 	}
 
 	for _, tc := range testCases {
-		st := state.NewManager(log.New("test_stale_results_handler"), testMetrics.GetStateMetrics(), dbstore, dbstore)
+		st := state.NewManager(log.New("test_stale_results_handler"), testMetrics.GetStateMetrics(), nil, dbstore, dbstore)
 		st.Warm()
 		existingStatesForRule := st.GetStatesForRuleUID(rule.OrgID, rule.UID)
 

pkg/services/ngalert/state/template.go

@@ -0,0 +1,89 @@
+package state
+
+import (
+	"context"
+	"math"
+	"net/url"
+	"strconv"
+	"time"
+
+	text_template "text/template"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	"github.com/prometheus/common/model"
+	"github.com/prometheus/prometheus/pkg/timestamp"
+	"github.com/prometheus/prometheus/promql"
+	"github.com/prometheus/prometheus/template"
+)
+
+// templateCaptureValue represents each value in .Values in the annotations
+// and labels template.
+type templateCaptureValue struct {
+	Labels map[string]string
+	Value  float64
+}
+
+// String implements the Stringer interface to print the value of each RefID
+// in the template via {{ $values.A }} rather than {{ $values.A.Value }}.
+func (v templateCaptureValue) String() string {
+	return strconv.FormatFloat(v.Value, 'f', -1, 64)
+}
+
+func expandTemplate(name, text string, labels map[string]string, alertInstance eval.Result, externalURL *url.URL) (result string, resultErr error) {
+	name = "__alert_" + name
+	text = "{{- $labels := .Labels -}}{{- $values := .Values -}}{{- $value := .Value -}}" + text
+	data := struct {
+		Labels map[string]string
+		Values map[string]templateCaptureValue
+		Value  string
+	}{
+		Labels: labels,
+		Values: newTemplateCaptureValues(alertInstance.Values),
+		Value:  alertInstance.EvaluationString,
+	}
+
+	expander := template.NewTemplateExpander(
+		context.TODO(), // This context is only used with the `query()` function - which we don't support yet.
+		text,
+		name,
+		data,
+		model.Time(timestamp.FromTime(alertInstance.EvaluatedAt)),
+		func(context.Context, string, time.Time) (promql.Vector, error) {
+			return nil, nil
+		},
+		externalURL,
+		[]string{"missingkey=error"},
+	)
+
+	expander.Funcs(text_template.FuncMap{
+		// These three functions are no-ops for now.
+		"strvalue": func(value templateCaptureValue) string {
+			return ""
+		},
+		"graphLink": func() string {
+			return ""
+		},
+		"tableLink": func() string {
+			return ""
+		},
+	})
+
+	return expander.Expand()
+}
+
+func newTemplateCaptureValues(values map[string]eval.NumberValueCapture) map[string]templateCaptureValue {
+	m := make(map[string]templateCaptureValue)
+	for k, v := range values {
+		var f float64
+		if v.Value != nil {
+			f = *v.Value
+		} else {
+			f = math.NaN()
+		}
+		m[k] = templateCaptureValue{
+			Labels: v.Labels,
+			Value:  f,
+		}
+	}
+	return m
+}

pkg/services/ngalert/state/template_test.go

@@ -0,0 +1,400 @@
+package state
+
+import (
+	"errors"
+	"net/url"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	ptr "github.com/xorcare/pointer"
+)
+
+func TestTemplateCaptureValueStringer(t *testing.T) {
+	cases := []struct {
+		name     string
+		value    templateCaptureValue
+		expected string
+	}{{
+		name:     "0 is returned as integer value",
+		value:    templateCaptureValue{Value: 0},
+		expected: "0",
+	}, {
+		name:     "1.0 is returned as integer value",
+		value:    templateCaptureValue{Value: 1.0},
+		expected: "1",
+	}, {
+		name:     "1.1 is returned as decimal value",
+		value:    templateCaptureValue{Value: 1.1},
+		expected: "1.1",
+	}}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			assert.Equal(t, c.expected, c.value.String())
+		})
+	}
+}
+
+func TestExpandTemplate(t *testing.T) {
+	pathPrefix := "/path/prefix"
+	externalURL, err := url.Parse("http://localhost" + pathPrefix)
+	assert.NoError(t, err)
+
+	cases := []struct {
+		name          string
+		text          string
+		alertInstance eval.Result
+		labels        data.Labels
+		expected      string
+		expectedError error
+	}{{
+		name:     "labels are expanded into $labels",
+		text:     "{{ $labels.instance }} is down",
+		labels:   data.Labels{"instance": "foo"},
+		expected: "foo is down",
+	}, {
+		name:          "missing label in $labels returns error",
+		text:          "{{ $labels.instance }} is down",
+		labels:        data.Labels{},
+		expectedError: errors.New("error executing template __alert_test: template: __alert_test:1:86: executing \"__alert_test\" at <$labels.instance>: map has no entry for key \"instance\""),
+	}, {
+		name: "values are expanded into $values",
+		text: "{{ $values.A.Labels.instance }} has value {{ $values.A }}",
+		alertInstance: eval.Result{
+			Values: map[string]eval.NumberValueCapture{
+				"A": {
+					Var:    "A",
+					Labels: data.Labels{"instance": "foo"},
+					Value:  ptr.Float64(1),
+				},
+			},
+		},
+		expected: "foo has value 1",
+	}, {
+		name: "values can be passed to template functions such as printf",
+		text: "{{ $values.A.Labels.instance }} has value {{ $values.A.Value | printf \"%.1f\" }}",
+		alertInstance: eval.Result{
+			Values: map[string]eval.NumberValueCapture{
+				"A": {
+					Var:    "A",
+					Labels: data.Labels{"instance": "foo"},
+					Value:  ptr.Float64(1.1),
+				},
+			},
+		},
+		expected: "foo has value 1.1",
+	}, {
+		name: "missing label in $values returns error",
+		text: "{{ $values.A.Labels.instance }} has value {{ $values.A }}",
+		alertInstance: eval.Result{
+			Values: map[string]eval.NumberValueCapture{
+				"A": {
+					Var:    "A",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(1),
+				},
+			},
+		},
+		expectedError: errors.New("error executing template __alert_test: template: __alert_test:1:86: executing \"__alert_test\" at <$values.A.Labels.instance>: map has no entry for key \"instance\""),
+	}, {
+		name: "missing value in $values is returned as NaN",
+		text: "{{ $values.A.Labels.instance }} has value {{ $values.A }}",
+		alertInstance: eval.Result{
+			Values: map[string]eval.NumberValueCapture{
+				"A": {
+					Var:    "A",
+					Labels: data.Labels{"instance": "foo"},
+					Value:  nil,
+				},
+			},
+		},
+		expected: "foo has value NaN",
+	}, {
+		name: "assert value string is expanded into $value",
+		text: "{{ $value }}",
+		alertInstance: eval.Result{
+			EvaluationString: "[ var='A' labels={instance=foo} value=10 ]",
+		},
+		expected: "[ var='A' labels={instance=foo} value=10 ]",
+	}, {
+		name: "float64 is humanized correctly",
+		text: "{{ humanize $value }}",
+		alertInstance: eval.Result{
+			EvaluationString: "1234567.0",
+		},
+		expected: "1.235M",
+	}, {
+		name: "int is humanized correctly",
+		text: "{{ humanize $value }}",
+		alertInstance: eval.Result{
+			EvaluationString: "1234567",
+		},
+		expected: "1.235M",
+	}, {
+		name: "humanize string with error",
+		text: `{{ humanize $value }}`,
+		alertInstance: eval.Result{
+			EvaluationString: "invalid",
+		},
+		expectedError: errors.New(`error executing template __alert_test: template: __alert_test:1:79: executing "__alert_test" at <humanize $value>: error calling humanize: strconv.ParseFloat: parsing "invalid": invalid syntax`),
+	}, {
+		name: "humanize1024 float64",
+		text: "{{ range $key, $val := $values }}{{ humanize1024 .Value }}:{{ end }}",
+		alertInstance: eval.Result{
+			Values: map[string]eval.NumberValueCapture{
+				"A": {
+					Var:    "A",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(0.0),
+				},
+				"B": {
+					Var:    "B",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(1.0),
+				},
+				"C": {
+					Var:    "C",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(1048576.0),
+				},
+				"D": {
+					Var:    "D",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(.12),
+				},
+			},
+		},
+		expected: "0:1:1Mi:0.12:",
+	}, {
+		name: "humanize1024 string with error",
+		text: "{{ humanize1024 $value }}",
+		alertInstance: eval.Result{
+			EvaluationString: "invalid",
+		},
+		expectedError: errors.New(`error executing template __alert_test: template: __alert_test:1:79: executing "__alert_test" at <humanize1024 $value>: error calling humanize1024: strconv.ParseFloat: parsing "invalid": invalid syntax`),
+	}, {
+		name: "humanizeDuration - seconds - float64",
+		text: "{{ range $key, $val := $values }}{{ humanizeDuration .Value }}:{{ end }}",
+		alertInstance: eval.Result{
+			Values: map[string]eval.NumberValueCapture{
+				"A": {
+					Var:    "A",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(0),
+				},
+				"B": {
+					Var:    "B",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(1),
+				},
+				"C": {
+					Var:    "C",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(60),
+				},
+				"D": {
+					Var:    "D",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(3600),
+				},
+				"E": {
+					Var:    "E",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(86400),
+				},
+				"F": {
+					Var:    "F",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(86400 + 3600),
+				},
+				"G": {
+					Var:    "G",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(-(86400*2 + 3600*3 + 60*4 + 5)),
+				},
+				"H": {
+					Var:    "H",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(899.99),
+				},
+			},
+		},
+		expected: "0s:1s:1m 0s:1h 0m 0s:1d 0h 0m 0s:1d 1h 0m 0s:-2d 3h 4m 5s:14m 59s:",
+	}, {
+		name: "humanizeDuration - string",
+		text: "{{ humanizeDuration $value }}",
+		alertInstance: eval.Result{
+			EvaluationString: "86400",
+		},
+		expected: "1d 0h 0m 0s",
+	}, {
+		name: "humanizeDuration - subsecond and fractional seconds - float64",
+		text: "{{ range $key, $val := $values }}{{ humanizeDuration .Value }}:{{ end }}",
+		alertInstance: eval.Result{
+			Values: map[string]eval.NumberValueCapture{
+				"A": {
+					Var:    "A",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(.1),
+				},
+				"B": {
+					Var:    "B",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(.0001),
+				},
+				"C": {
+					Var:    "C",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(.12345),
+				},
+				"D": {
+					Var:    "D",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(60.1),
+				},
+				"E": {
+					Var:    "E",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(60.5),
+				},
+				"F": {
+					Var:    "F",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(1.2345),
+				},
+				"G": {
+					Var:    "G",
+					Labels: data.Labels{},
+					Value:  ptr.Float64(12.345),
+				},
+			},
+		},
+		expected: "100ms:100us:123.5ms:1m 0s:1m 0s:1.234s:12.35s:",
+	}, {
+		name: "humanizeDuration - subsecond - string",
+		text: "{{ humanizeDuration $value }}",
+		alertInstance: eval.Result{
+			EvaluationString: ".0001",
+		},
+		expected: "100us",
+	}, {
+		name: "humanizeDuration - fractional seconds - string",
+		text: "{{ humanizeDuration $value }}",
+		alertInstance: eval.Result{
+			EvaluationString: "1.2345",
+		},
+		expected: "1.234s",
+	}, {
+		name: "humanizeDuration - string with error",
+		text: `{{ humanizeDuration $value }}`,
+		alertInstance: eval.Result{
+			EvaluationString: "invalid",
+		},
+		expectedError: errors.New(`error executing template __alert_test: template: __alert_test:1:79: executing "__alert_test" at <humanizeDuration $value>: error calling humanizeDuration: strconv.ParseFloat: parsing "invalid": invalid syntax`),
+	}, {
+		name:     "humanizePercentage - float64",
+		text:     "{{ -0.22222 | humanizePercentage }}:{{ 0.0 | humanizePercentage }}:{{ 0.1234567 | humanizePercentage }}:{{ 1.23456 | humanizePercentage }}",
+		expected: "-22.22%:0%:12.35%:123.5%",
+	}, {
+		name:     "humanizePercentage - string",
+		text:     `{{ "-0.22222" | humanizePercentage }}:{{ "0.0" | humanizePercentage }}:{{ "0.1234567" | humanizePercentage }}:{{ "1.23456" | humanizePercentage }}`,
+		expected: "-22.22%:0%:12.35%:123.5%",
+	}, {
+		name:          "humanizePercentage - string with error",
+		text:          `{{ "invalid" | humanizePercentage }}`,
+		expectedError: errors.New(`error executing template __alert_test: template: __alert_test:1:91: executing "__alert_test" at <humanizePercentage>: error calling humanizePercentage: strconv.ParseFloat: parsing "invalid": invalid syntax`),
+	}, {
+		name:     "humanizeTimestamp - float64",
+		text:     "{{ 1435065584.128 | humanizeTimestamp }}",
+		expected: "2015-06-23 13:19:44.128 +0000 UTC",
+	}, {
+		name:     "humanizeTimestamp - string",
+		text:     `{{ "1435065584.128" | humanizeTimestamp }}`,
+		expected: "2015-06-23 13:19:44.128 +0000 UTC",
+	}, {
+		name:     "title",
+		text:     `{{ "aa bb CC" | title }}`,
+		expected: "Aa Bb CC",
+	}, {
+		name:     "toUpper",
+		text:     `{{ "aa bb CC" | toUpper }}`,
+		expected: "AA BB CC",
+	}, {
+		name:     "toLower",
+		text:     `{{ "aA bB CC" | toLower }}`,
+		expected: "aa bb cc",
+	}, {
+		name:     "match",
+		text:     `{{ match "a+" "aa" }} {{ match "a+" "b" }}`,
+		expected: "true false",
+	}, {
+		name:     "regex replacement",
+		text:     "{{ reReplaceAll \"(a)b\" \"x$1\" \"ab\" }}",
+		expected: "xa",
+	}, {
+		name:     "pass multiple arguments to templates",
+		text:     `{{define "x"}}{{.arg0}} {{.arg1}}{{end}}{{template "x" (args 1 "2")}}`,
+		expected: "1 2",
+	}, {
+		name:     "pathPrefix",
+		text:     "{{ pathPrefix }}",
+		expected: pathPrefix,
+	}, {
+		name:     "externalURL",
+		text:     "{{ externalURL }}",
+		expected: externalURL.String(),
+	}, {
+		name:     "check that query, first and value don't error or panic",
+		text:     "{{ query \"1.5\" | first | value }}",
+		expected: "",
+	}, {
+		name:     "check that label doesn't error or panic",
+		text:     "{{ query \"metric{instance='a'}\" | first | label \"instance\" }}",
+		expected: "",
+	}, {
+		name:     "check that graphLink returns an empty string",
+		text:     "{{ graphLink \"up\" }}",
+		expected: "",
+	}, {
+		name:     "check that tableLink returns an empty string",
+		text:     "{{ tableLink \"up\" }}",
+		expected: "",
+	}, {
+		name:     "check that sortByLabel doesn't error or panic",
+		text:     "{{ query \"metric{__value__='a'}\" | sortByLabel }}",
+		expected: "",
+	}, {
+		name: "check that strvalue returns an empty string (for now)",
+		text: "{{ $values.A | strvalue }}",
+		alertInstance: eval.Result{
+			Values: map[string]eval.NumberValueCapture{
+				"A": {
+					Var:    "A",
+					Labels: data.Labels{"__value__": "foo"},
+				},
+			},
+		},
+		expected: "",
+	}, {
+		name:     "check that safeHtml doesn't error or panic",
+		text:     "{{ \"<b>\" | safeHtml }}",
+		expected: "<b>",
+	},
+	}
+
+	for _, c := range cases {
+		t.Run(c.name, func(t *testing.T) {
+			v, err := expandTemplate("test", c.text, c.labels, c.alertInstance, externalURL)
+			if c.expectedError != nil {
+				require.NotNil(t, err)
+				require.EqualError(t, c.expectedError, err.Error())
+			} else {
+				require.Nil(t, c.expectedError)
+			}
+			require.Equal(t, c.expected, v)
+		})
+	}
+}