Alerting: Sanitize invalid label/annotation names for external alertmanagers (#54537)

* Alerting: Sanitize invalid label/annotation names for external alertmanagers

Grafana's built-in Alertmanager supports both Unicode label keys and values; however, if using an external
Prometheus Alertmanager label keys must be compatible with their data model.
This means label keys must only contain ASCII letters, numbers, as well as underscores and match the regex
`[a-zA-Z_][a-zA-Z0-9_]*`.

Any invalid characters will now be removed or replaced by the Grafana alerting engine before being sent to
the external Alertmanager according to the following rules:

- `Whitespace` will be removed.
- `ASCII characters` will be replaced with `_`.
- `All other characters` will be replaced with their lower-case hex representation.

* Prefix hex replacements with `0x`

* Refactor for clarity

* Apply suggestions from code review

Co-authored-by: brendamuir <100768211+brendamuir@users.noreply.github.com>

Co-authored-by: brendamuir <100768211+brendamuir@users.noreply.github.com>

docs/sources/alerting/fundamentals/annotation-label/how-to-use-labels.md

@@ -23,6 +23,20 @@ This topic explains why labels are a fundamental component of alerting.
 
 {{< figure src="/static/img/docs/alerting/unified/rule-edit-details-8-0.png" max-width="550px" caption="Alert details" >}}
 
+# External Alertmanager Compatibility
+
+Grafana's built-in Alertmanager supports both Unicode label keys and values. If you are using an external Prometheus Alertmanager, label keys must be compatible with their [data model](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels).
+This means that label keys must only contain **ASCII letters**, **numbers**, as well as **underscores** and match the regex `[a-zA-Z_][a-zA-Z0-9_]*`.
+Any invalid characters will be removed or replaced by the Grafana alerting engine before being sent to the external Alertmanager according to the following rules:
+
+- `Whitespace` will be removed.
+- `ASCII characters` will be replaced with `_`.
+- `All other characters` will be replaced with their lower-case hex representation. If this is the first character it will be prefixed with `_`.
+
+Example: A label key/value pair `Alert! 🔔="🔥"` will become `Alert_0x1f514="🔥"`.
+
+**Note** If multiple label keys are sanitized to the same value, the duplicates will have a short hash of the original label appended as a suffix.
+
 # Grafana reserved labels
 
 > **Note:** Labels prefixed with `grafana_` are reserved by Grafana for special use. If a manually configured label is added beginning with `grafana_` it may be overwritten in case of collision.

pkg/services/ngalert/sender/sender.go

@@ -2,10 +2,15 @@ package sender
 
 import (
 	"context"
+	"crypto/md5"
+	"errors"
+	"fmt"
 	"net/url"
+	"sort"
 	"strings"
 	"sync"
 	"time"
+	"unicode"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
@@ -100,7 +105,7 @@ func (s *ExternalAlertmanager) SendAlerts(alerts apimodels.PostableAlerts) {
 	}
 	as := make([]*notifier.Alert, 0, len(alerts.PostableAlerts))
 	for _, a := range alerts.PostableAlerts {
-		na := alertToNotifierAlert(a)
+		na := s.alertToNotifierAlert(a)
 		as = append(as, na)
 	}
 
@@ -171,29 +176,114 @@ func buildNotifierConfig(cfg *ngmodels.AdminConfiguration) (*config.Config, erro
 	return notifierConfig, nil
 }
 
-func alertToNotifierAlert(alert models.PostableAlert) *notifier.Alert {
-	ls := make(labels.Labels, 0, len(alert.Alert.Labels))
-	a := make(labels.Labels, 0, len(alert.Annotations))
-
-	// Prometheus does not allow spaces in labels or annotations while Grafana does, we need to make sure we
-	// remove them before sending the alerts.
-	for k, v := range alert.Alert.Labels {
-		ls = append(ls, labels.Label{Name: removeSpaces(k), Value: v})
-	}
-
-	for k, v := range alert.Annotations {
-		a = append(a, labels.Label{Name: removeSpaces(k), Value: v})
-	}
-
+func (s *ExternalAlertmanager) alertToNotifierAlert(alert models.PostableAlert) *notifier.Alert {
+	// Prometheus alertmanager has stricter rules for annotations/labels than grafana's internal alertmanager, so we sanitize invalid keys.
 	return &notifier.Alert{
-		Labels:       ls,
-		Annotations:  a,
+		Labels:       s.sanitizeLabelSet(alert.Alert.Labels),
+		Annotations:  s.sanitizeLabelSet(alert.Annotations),
 		StartsAt:     time.Time(alert.StartsAt),
 		EndsAt:       time.Time(alert.EndsAt),
 		GeneratorURL: alert.Alert.GeneratorURL.String(),
 	}
 }
 
-func removeSpaces(labelName string) string {
-	return strings.Join(strings.Fields(labelName), "")
+// sanitizeLabelSet sanitizes all given LabelSet keys according to sanitizeLabelName.
+// If there is a collision as a result of sanitization, a short (6 char) md5 hash of the original key will be added as a suffix.
+func (s *ExternalAlertmanager) sanitizeLabelSet(lbls models.LabelSet) labels.Labels {
+	ls := make(labels.Labels, 0, len(lbls))
+	set := make(map[string]struct{})
+
+	// Must sanitize labels in order otherwise resulting label set can be inconsistent when there are collisions.
+	for _, k := range sortedKeys(lbls) {
+		sanitizedLabelName, err := s.sanitizeLabelName(k)
+		if err != nil {
+			s.logger.Error("alert sending to external Alertmanager(s) contains an invalid label/annotation name that failed to sanitize, skipping", "name", k, "err", err)
+			continue
+		}
+
+		// There can be label name collisions after we sanitize. We check for this and attempt to make the name unique again using a short hash of the original name.
+		if _, ok := set[sanitizedLabelName]; ok {
+			sanitizedLabelName = sanitizedLabelName + fmt.Sprintf("_%.3x", md5.Sum([]byte(k)))
+			s.logger.Warn("alert contains duplicate label/annotation name after sanitization, appending unique suffix", "name", k, "new_name", sanitizedLabelName, "err", err)
+		}
+
+		set[sanitizedLabelName] = struct{}{}
+		ls = append(ls, labels.Label{Name: sanitizedLabelName, Value: lbls[k]})
+	}
+
+	return ls
+}
+
+// sanitizeLabelName will fix a given label name so that it is compatible with prometheus alertmanager character restrictions.
+// Prometheus alertmanager requires labels to match ^[a-zA-Z_][a-zA-Z0-9_]*$.
+// Characters with an ASCII code < 127 will be replaced with an underscore (_), characters with ASCII code >= 127 will be replaced by their hex representation.
+// For backwards compatibility, whitespace will be removed instead of replaced with an underscore.
+func (s *ExternalAlertmanager) sanitizeLabelName(name string) (string, error) {
+	if len(name) == 0 {
+		return "", errors.New("label name cannot be empty")
+	}
+
+	if isValidLabelName(name) {
+		return name, nil
+	}
+
+	s.logger.Warn("alert sending to external Alertmanager(s) contains label/annotation name with invalid characters", "name", name)
+
+	// Remove spaces. We do this instead of replacing with underscore for backwards compatibility as this existed before the rest of this function.
+	sanitized := strings.Join(strings.Fields(name), "")
+
+	// Replace other invalid characters.
+	var buf strings.Builder
+	for i, b := range sanitized {
+		if isValidCharacter(i, b) {
+			buf.WriteRune(b)
+			continue
+		}
+
+		if b <= unicode.MaxASCII {
+			buf.WriteRune('_')
+			continue
+		}
+
+		if i == 0 {
+			buf.WriteRune('_')
+		}
+		_, _ = fmt.Fprintf(&buf, "%#x", b)
+	}
+
+	if buf.Len() == 0 {
+		return "", fmt.Errorf("label name is empty after removing invalids chars")
+	}
+
+	return buf.String(), nil
+}
+
+// isValidLabelName is true iff the label name matches the pattern of ^[a-zA-Z_][a-zA-Z0-9_]*$.
+func isValidLabelName(ln string) bool {
+	if len(ln) == 0 {
+		return false
+	}
+	for i, b := range ln {
+		if !isValidCharacter(i, b) {
+			return false
+		}
+	}
+	return true
+}
+
+// isValidCharacter checks if a specific rune is allowed at the given position in a label key for an external Prometheus alertmanager.
+// From alertmanager LabelName.IsValid().
+func isValidCharacter(pos int, b rune) bool {
+	return (b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z') || b == '_' || (b >= '0' && b <= '9' && pos > 0)
+}
+
+func sortedKeys(m map[string]string) []string {
+	orderedKeys := make([]string, len(m))
+	i := 0
+	for k := range m {
+		orderedKeys[i] = k
+		i++
+	}
+	sort.Strings(orderedKeys)
+	return orderedKeys
 }

pkg/services/ngalert/sender/sender_test.go

@@ -0,0 +1,103 @@
+package sender
+
+import (
+	"testing"
+
+	"github.com/prometheus/alertmanager/api/v2/models"
+	"github.com/prometheus/prometheus/pkg/labels"
+	"github.com/stretchr/testify/require"
+)
+
+func TestSanitizeLabelName(t *testing.T) {
+	cases := []struct {
+		desc           string
+		labelName      string
+		expectedResult string
+		expectedErr    string
+	}{
+		{
+			desc:           "Remove whitespace",
+			labelName:      "   a\tb\nc\vd\re\ff   ",
+			expectedResult: "abcdef",
+		},
+		{
+			desc:           "Replace ASCII with underscore",
+			labelName:      " !\"#$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~",
+			expectedResult: "________________0123456789_______ABCDEFGHIJKLMNOPQRSTUVWXYZ______abcdefghijklmnopqrstuvwxyz____",
+		},
+		{
+			desc:           "Replace non-ASCII unicode with hex",
+			labelName:      "_€_ƒ_„_†_‡_œ_Ÿ_®_º_¼_×_ð_þ_¿_±_四_十_二_🔥",
+			expectedResult: "_0x20ac_0x192_0x201e_0x2020_0x2021_0x153_0x178_0xae_0xba_0xbc_0xd7_0xf0_0xfe_0xbf_0xb1_0x56db_0x5341_0x4e8c_0x1f525",
+		},
+		{ // labels starting with a number are invalid, so we have to make sure we don't sanitize to another invalid label.
+			desc:           "If first character is replaced with hex, prefix with underscore",
+			labelName:      "😍😍😍",
+			expectedResult: "_0x1f60d0x1f60d0x1f60d",
+		},
+		{
+			desc:        "Empty string should error",
+			labelName:   "",
+			expectedErr: "label name cannot be empty",
+		},
+		{
+			desc:        "Only whitespace should error",
+			labelName:   "   \t\n\v\n\f   ",
+			expectedErr: "label name is empty after removing invalids chars",
+		},
+	}
+
+	for _, tc := range cases {
+		am, _ := NewExternalAlertmanagerSender()
+		t.Run(tc.desc, func(t *testing.T) {
+			res, err := am.sanitizeLabelName(tc.labelName)
+
+			if tc.expectedErr != "" {
+				require.EqualError(t, err, tc.expectedErr)
+			}
+
+			require.Equal(t, tc.expectedResult, res)
+		})
+	}
+}
+
+func TestSanitizeLabelSet(t *testing.T) {
+	cases := []struct {
+		desc           string
+		labelset       models.LabelSet
+		expectedResult labels.Labels
+	}{
+		{
+			desc: "Duplicate labels after sanitizations, append short has as suffix to duplicates",
+			labelset: models.LabelSet{
+				"test-alert": "42",
+				"test_alert": "43",
+				"test+alert": "44",
+			},
+			expectedResult: labels.Labels{
+				labels.Label{Name: "test_alert", Value: "44"},
+				labels.Label{Name: "test_alert_ed6237", Value: "42"},
+				labels.Label{Name: "test_alert_a67b5e", Value: "43"},
+			},
+		},
+		{
+			desc: "If sanitize fails for a label, skip it",
+			labelset: models.LabelSet{
+				"test-alert":       "42",
+				"   \t\n\v\n\f   ": "43",
+				"test+alert":       "44",
+			},
+			expectedResult: labels.Labels{
+				labels.Label{Name: "test_alert", Value: "44"},
+				labels.Label{Name: "test_alert_ed6237", Value: "42"},
+			},
+		},
+	}
+
+	for _, tc := range cases {
+		am, _ := NewExternalAlertmanagerSender()
+		t.Run(tc.desc, func(t *testing.T) {
+			require.Equal(t, tc.expectedResult, am.sanitizeLabelSet(tc.labelset))
+		})
+	}
+}