Alerting: Use remote Alertmanager to test templates and receivers when enabled (#91570)

* Initial impl

* Add code to test templates and receivers

* Fix linter

* Fix forked am tests

* Update mimir client

* Remove trailing whitespace

* re-trigger CI

pkg/services/ngalert/notifier/templates.go

@@ -31,7 +31,7 @@ var (
 // If an existing template of the same filename as the one being tested is found, it will not be used as context.
 func (am *alertmanager) TestTemplate(ctx context.Context, c apimodels.TestTemplatesConfigBodyParams) (*TestTemplatesResults, error) {
 	for _, alert := range c.Alerts {
-		addDefaultLabelsAndAnnotations(alert)
+		AddDefaultLabelsAndAnnotations(alert)
 	}
 
 	return am.Base.TestTemplate(ctx, alertingNotify.TestTemplatesConfigBodyParams{
@@ -41,8 +41,8 @@ func (am *alertmanager) TestTemplate(ctx context.Context, c apimodels.TestTempla
 	})
 }
 
-// addDefaultLabelsAndAnnotations is a slimmed down version of state.StateToPostableAlert and state.GetRuleExtraLabels using default values.
+// AddDefaultLabelsAndAnnotations is a slimmed down version of state.StateToPostableAlert and state.GetRuleExtraLabels using default values.
-func addDefaultLabelsAndAnnotations(alert *amv2.PostableAlert) {
+func AddDefaultLabelsAndAnnotations(alert *amv2.PostableAlert) {
 	if alert.Labels == nil {
 		alert.Labels = make(map[string]string)
 	}

pkg/services/ngalert/remote/alertmanager.go

@@ -510,11 +510,47 @@ func (am *Alertmanager) GetReceivers(ctx context.Context) ([]apimodels.Receiver,
 }
 
 func (am *Alertmanager) TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*alertingNotify.TestReceiversResult, int, error) {
-	return &alertingNotify.TestReceiversResult{}, 0, nil
+	receivers := make([]*alertingNotify.APIReceiver, 0, len(c.Receivers))
+	for _, r := range c.Receivers {
+		integrations := make([]*alertingNotify.GrafanaIntegrationConfig, 0, len(r.GrafanaManagedReceivers))
+		for _, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
+			integrations = append(integrations, &alertingNotify.GrafanaIntegrationConfig{
+				UID:                   gr.UID,
+				Name:                  gr.Name,
+				Type:                  gr.Type,
+				DisableResolveMessage: gr.DisableResolveMessage,
+				Settings:              json.RawMessage(gr.Settings),
+				SecureSettings:        gr.SecureSettings,
+			})
+		}
+		receivers = append(receivers, &alertingNotify.APIReceiver{
+			ConfigReceiver: r.Receiver,
+			GrafanaIntegrations: alertingNotify.GrafanaIntegrations{
+				Integrations: integrations,
+			},
+		})
+	}
+	var alert *alertingNotify.TestReceiversConfigAlertParams
+	if c.Alert != nil {
+		alert = &alertingNotify.TestReceiversConfigAlertParams{Annotations: c.Alert.Annotations, Labels: c.Alert.Labels}
+	}
+
+	return am.mimirClient.TestReceivers(ctx, alertingNotify.TestReceiversConfigBodyParams{
+		Alert:     alert,
+		Receivers: receivers,
+	})
 }
 
 func (am *Alertmanager) TestTemplate(ctx context.Context, c apimodels.TestTemplatesConfigBodyParams) (*notifier.TestTemplatesResults, error) {
-	return &notifier.TestTemplatesResults{}, nil
+	for _, alert := range c.Alerts {
+		notifier.AddDefaultLabelsAndAnnotations(alert)
+	}
+
+	return am.mimirClient.TestTemplate(ctx, alertingNotify.TestTemplatesConfigBodyParams{
+		Alerts:   c.Alerts,
+		Template: c.Template,
+		Name:     c.Name,
+	})
 }
 
 // StopAndWait is called when the grafana server is instructed to shut down or an org is deleted.

pkg/services/ngalert/remote/client/mimir.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"errors"
@@ -11,6 +12,7 @@ import (
 	"path"
 	"strings"
 
+	alertingNotify "github.com/grafana/alerting/notify"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
@@ -28,6 +30,9 @@ type MimirClient interface {
 	CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration *apimodels.PostableUserConfig, hash string, createdAt int64, isDefault bool) error
 	DeleteGrafanaAlertmanagerConfig(ctx context.Context) error
 
+	TestTemplate(ctx context.Context, c alertingNotify.TestTemplatesConfigBodyParams) (*alertingNotify.TestTemplatesResults, error)
+	TestReceivers(ctx context.Context, c alertingNotify.TestReceiversConfigBodyParams) (*alertingNotify.TestReceiversResult, int, error)
+
 	ShouldPromoteConfig() bool
 
 	// Mimir implements an extended version of the receivers API under a different path.
@@ -194,3 +199,39 @@ func (mc *Mimir) doOK(ctx context.Context, p, method string, payload io.Reader)
 		return fmt.Errorf("received an unknown status from the request body: %s", sr.Status)
 	}
 }
+
+func (mc *Mimir) TestReceivers(ctx context.Context, c alertingNotify.TestReceiversConfigBodyParams) (*alertingNotify.TestReceiversResult, int, error) {
+	payload, err := json.Marshal(c)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	trResult := &alertingNotify.TestReceiversResult{}
+
+	// nolint:bodyclose
+	// closed within `do`
+	_, err = mc.do(ctx, "api/v1/grafana/receivers/test", http.MethodPost, bytes.NewBuffer(payload), &trResult)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	return trResult, http.StatusOK, nil
+}
+
+func (mc *Mimir) TestTemplate(ctx context.Context, c alertingNotify.TestTemplatesConfigBodyParams) (*alertingNotify.TestTemplatesResults, error) {
+	payload, err := json.Marshal(c)
+	if err != nil {
+		return nil, err
+	}
+
+	ttResult := &alertingNotify.TestTemplatesResults{}
+
+	// nolint:bodyclose
+	// closed within `do`
+	_, err = mc.do(ctx, "api/v1/grafana/templates/test", http.MethodPost, bytes.NewBuffer(payload), &ttResult)
+	if err != nil {
+		return nil, err
+	}
+
+	return ttResult, nil
+}

pkg/services/ngalert/remote/forked_alertmanager_test.go

@@ -653,30 +653,28 @@ func TestForkedAlertmanager_ModeRemotePrimary(t *testing.T) {
 
 	t.Run("TestReceivers", func(tt *testing.T) {
 		// TestReceivers should be called only in the remote Alertmanager.
-		// TODO: change to remote AM once it's implemented there.
+		_, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary)
-		internal, _, forked := genTestAlertmanagers(tt, modeRemotePrimary)
+		remote.EXPECT().TestReceivers(mock.Anything, mock.Anything).Return(nil, 0, nil).Once()
-		internal.EXPECT().TestReceivers(mock.Anything, mock.Anything).Return(nil, 0, nil).Once()
 		_, _, err := forked.TestReceivers(ctx, apimodels.TestReceiversConfigBodyParams{})
 		require.NoError(tt, err)
 
 		// If there's an error in the remote Alertmanager, it should be returned.
-		internal, _, forked = genTestAlertmanagers(tt, modeRemotePrimary)
+		_, remote, forked = genTestAlertmanagers(tt, modeRemotePrimary)
-		internal.EXPECT().TestReceivers(mock.Anything, mock.Anything).Return(nil, 0, expErr).Once()
+		remote.EXPECT().TestReceivers(mock.Anything, mock.Anything).Return(nil, 0, expErr).Once()
 		_, _, err = forked.TestReceivers(ctx, apimodels.TestReceiversConfigBodyParams{})
 		require.ErrorIs(tt, expErr, err)
 	})
 
 	t.Run("TestTemplate", func(tt *testing.T) {
 		// TestTemplate should be called only in the internal Alertmanager.
-		// TODO: change to remote AM once it's implemented there.
+		_, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary)
-		internal, _, forked := genTestAlertmanagers(tt, modeRemotePrimary)
+		remote.EXPECT().TestTemplate(mock.Anything, mock.Anything).Return(nil, nil).Once()
-		internal.EXPECT().TestTemplate(mock.Anything, mock.Anything).Return(nil, nil).Once()
 		_, err := forked.TestTemplate(ctx, apimodels.TestTemplatesConfigBodyParams{})
 		require.NoError(tt, err)
 
 		// If there's an error in the internal Alertmanager, it should be returned.
-		internal, _, forked = genTestAlertmanagers(tt, modeRemotePrimary)
+		_, remote, forked = genTestAlertmanagers(tt, modeRemotePrimary)
-		internal.EXPECT().TestTemplate(mock.Anything, mock.Anything).Return(nil, expErr).Once()
+		remote.EXPECT().TestTemplate(mock.Anything, mock.Anything).Return(nil, expErr).Once()
 		_, err = forked.TestTemplate(ctx, apimodels.TestTemplatesConfigBodyParams{})
 		require.ErrorIs(tt, expErr, err)
 	})

pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go

@@ -134,13 +134,11 @@ func (fam *RemotePrimaryForkedAlertmanager) GetReceivers(ctx context.Context) ([
 }
 
 func (fam *RemotePrimaryForkedAlertmanager) TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*alertingNotify.TestReceiversResult, int, error) {
-	// TODO: change to remote AM once it's implemented there.
+	return fam.remote.TestReceivers(ctx, c)
-	return fam.internal.TestReceivers(ctx, c)
 }
 
 func (fam *RemotePrimaryForkedAlertmanager) TestTemplate(ctx context.Context, c apimodels.TestTemplatesConfigBodyParams) (*notifier.TestTemplatesResults, error) {
-	// TODO: change to remote AM once it's implemented there.
+	return fam.remote.TestTemplate(ctx, c)
-	return fam.internal.TestTemplate(ctx, c)
 }
 
 func (fam *RemotePrimaryForkedAlertmanager) SilenceState(ctx context.Context) (alertingNotify.SilenceState, error) {