Alerting: Skip sanitizing labels when sending alerts to the remote Alertmanager (#96251)

* Alerting: Skip sanitizing labels when sending alerts to the remote Alertmanager

* fix drone image name

pkg/services/ngalert/remote/alertmanager.go

@@ -150,7 +150,12 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt
 		return c.Do(req.WithContext(ctx))
 	}
 	senderLogger := log.New("ngalert.sender.external-alertmanager")
-	s, err := sender.NewExternalAlertmanagerSender(senderLogger, prometheus.NewRegistry(), sender.WithDoFunc(doFunc))
+	s, err := sender.NewExternalAlertmanagerSender(
+		senderLogger,
+		prometheus.NewRegistry(),
+		sender.WithDoFunc(doFunc),
+		sender.WithUTF8Labels(),
+	)
 	if err != nil {
 		return nil, err
 	}

pkg/services/ngalert/remote/alertmanager_test.go

@@ -785,16 +785,18 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
 	require.Equal(t, 0, len(alertGroups))
 
 	// Let's create two active alerts and one expired one.
-	alert1 := genAlert(true, map[string]string{"test_1": "test_1", "empty": "", alertingModels.NamespaceUIDLabel: "test_1"})
-	alert2 := genAlert(true, map[string]string{"test_2": "test_2", "empty": "", alertingModels.NamespaceUIDLabel: "test_2"})
-	alert3 := genAlert(false, map[string]string{"test_3": "test_3", "empty": "", alertingModels.NamespaceUIDLabel: "test_3"})
+	// UTF-8 label names should be preserved.
+	utf8LabelName := "test utf-8 label 😳"
+	alert1 := genAlert(true, map[string]string{utf8LabelName: "test_1", "empty": "", alertingModels.NamespaceUIDLabel: "test_1"})
+	alert2 := genAlert(true, map[string]string{utf8LabelName: "test_2", "empty": "", alertingModels.NamespaceUIDLabel: "test_2"})
+	alert3 := genAlert(false, map[string]string{utf8LabelName: "test_3", "empty": "", alertingModels.NamespaceUIDLabel: "test_3"})
 	postableAlerts := apimodels.PostableAlerts{
 		PostableAlerts: []amv2.PostableAlert{alert1, alert2, alert3},
 	}
 	err = am.PutAlerts(context.Background(), postableAlerts)
 	require.NoError(t, err)
 
-	// We should have two alerts and one group now.
+	// We should eventually have two active alerts.
 	require.Eventually(t, func() bool {
 		alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{}, "")
 		require.NoError(t, err)
@@ -806,15 +808,15 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
 	require.Equal(t, 1, len(alertGroups))
 
 	// Labels with empty values and the namespace UID label should be removed.
+	// UTF-8 label names should remain unchanged.
 	for _, a := range alertGroups {
-		require.NotContains(t, a.Labels, "empty")
-		require.NotContains(t, a.Labels, alertingModels.NamespaceUIDLabel)
+		require.Len(t, a.Alerts, 2)
+		for _, a := range a.Alerts {
+			require.NotContains(t, a.Labels, "empty")
+			require.NotContains(t, a.Labels, alertingModels.NamespaceUIDLabel)
+			require.Contains(t, a.Labels, utf8LabelName)
+		}
 	}
-
-	// Filtering by `test_1=test_1` should return one alert.
-	alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{"test_1=test_1"}, "")
-	require.NoError(t, err)
-	require.Equal(t, 1, len(alerts))
 }
 
 func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) {

pkg/services/ngalert/sender/sender.go

@@ -37,8 +37,9 @@ type ExternalAlertmanager struct {
 
 	manager *Manager
 
-	sdCancel  context.CancelFunc
-	sdManager *discovery.Manager
+	sanitizeLabelSetFn func(lbls models.LabelSet) labels.Labels
+	sdCancel           context.CancelFunc
+	sdManager          *discovery.Manager
 }
 
 type ExternalAMcfg struct {
@@ -57,6 +58,20 @@ func WithDoFunc(doFunc doFunc) Option {
 	}
 }
 
+// WithUTF8Labels skips sanitizing labels and annotations before sending alerts to the external Alertmanager(s).
+// It assumes UTF-8 label names are supported by the Alertmanager(s).
+func WithUTF8Labels() Option {
+	return func(s *ExternalAlertmanager) {
+		s.sanitizeLabelSetFn = func(lbls models.LabelSet) labels.Labels {
+			ls := make(labels.Labels, 0, len(lbls))
+			for k, v := range lbls {
+				ls = append(ls, labels.Label{Name: k, Value: v})
+			}
+			return ls
+		}
+	}
+}
+
 func (cfg *ExternalAMcfg) SHA256() string {
 	return asSHA256([]string{cfg.headerString(), cfg.URL})
 }
@@ -87,6 +102,7 @@ func NewExternalAlertmanagerSender(l log.Logger, reg prometheus.Registerer, opts
 		sdCancel: sdCancel,
 	}
 
+	s.sanitizeLabelSetFn = s.sanitizeLabelSet
 	s.manager = NewManager(
 		// Injecting a new registry here means these metrics are not exported.
 		// Once we fix the individual Alertmanager metrics we should fix this scenario too.
@@ -240,8 +256,8 @@ func buildNotifierConfig(alertmanagers []ExternalAMcfg) (*config.Config, map[str
 func (s *ExternalAlertmanager) alertToNotifierAlert(alert models.PostableAlert) *Alert {
 	// Prometheus alertmanager has stricter rules for annotations/labels than grafana's internal alertmanager, so we sanitize invalid keys.
 	return &Alert{
-		Labels:       s.sanitizeLabelSet(alert.Alert.Labels),
-		Annotations:  s.sanitizeLabelSet(alert.Annotations),
+		Labels:       s.sanitizeLabelSetFn(alert.Alert.Labels),
+		Annotations:  s.sanitizeLabelSetFn(alert.Annotations),
 		StartsAt:     time.Time(alert.StartsAt),
 		EndsAt:       time.Time(alert.EndsAt),
 		GeneratorURL: alert.Alert.GeneratorURL.String(),