Contact point testing (#37308)

This commit adds contact point testing to ngalerts via a new API
endpoint. This endpoint accepts JSON containing a list of
receiver configurations which are validated and then tested
with a notification for a test alert. The endpoint returns JSON
for each receiver with a status and error message. It accepts
a configurable timeout via the Request-Timeout header (in seconds)
up to a maximum of 30 seconds.

pkg/services/ngalert/api/api.go

@@ -1,6 +1,7 @@
 package api
 
 import (
+	"context"
 	"net/url"
 	"time"
 
@@ -10,6 +11,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/datasources"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
+	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/ngalert/schedule"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -43,6 +45,9 @@ type Alertmanager interface {
 	// Alerts
 	GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error)
 	GetAlertGroups(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error)
+
+	// Testing
+	TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigParams) (*notifier.TestReceiversResult, error)
 }
 
 // API handlers.

pkg/services/ngalert/api/api_alertmanager.go

@@ -1,9 +1,13 @@
 package api
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"net/http"
+	"strconv"
+	"strings"
+	"time"
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -15,12 +19,79 @@ import (
 	"github.com/grafana/grafana/pkg/util"
 )
 
+const (
+	defaultTestReceiversTimeout = 15 * time.Second
+	maxTestReceiversTimeout     = 30 * time.Second
+)
+
 type AlertmanagerSrv struct {
 	am    Alertmanager
 	store store.AlertingStore
 	log   log.Logger
 }
 
+type UnknownReceiverError struct {
+	UID string
+}
+
+func (e UnknownReceiverError) Error() string {
+	return fmt.Sprintf("unknown receiver: %s", e.UID)
+}
+
+func (srv AlertmanagerSrv) loadSecureSettings(orgId int64, receivers []*apimodels.PostableApiReceiver) error {
+	// Get the last known working configuration
+	query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
+	if err := srv.store.GetLatestAlertmanagerConfiguration(&query); err != nil {
+		// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
+		if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
+			return fmt.Errorf("failed to get latest configuration: %w", err)
+		}
+	}
+
+	currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
+	if query.Result != nil {
+		currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
+		if err != nil {
+			return fmt.Errorf("failed to load latest configuration: %w", err)
+		}
+		currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
+	}
+
+	// Copy the previously known secure settings
+	for i, r := range receivers {
+		for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
+			if gr.UID == "" { // new receiver
+				continue
+			}
+
+			cgmr, ok := currentReceiverMap[gr.UID]
+			if !ok {
+				// it tries to update a receiver that didn't previously exist
+				return UnknownReceiverError{UID: gr.UID}
+			}
+
+			// frontend sends only the secure settings that have to be updated
+			// therefore we have to copy from the last configuration only those secure settings not included in the request
+			for key := range cgmr.SecureSettings {
+				_, ok := gr.SecureSettings[key]
+				if !ok {
+					decryptedValue, err := cgmr.GetDecryptedSecret(key)
+					if err != nil {
+						return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
+					}
+
+					if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
+						receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
+					}
+
+					receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func (srv AlertmanagerSrv) RouteGetAMStatus(c *models.ReqContext) response.Response {
 	return response.JSON(http.StatusOK, srv.am.GetStatus())
 }
@@ -210,46 +281,12 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 		}
 	}
 
-	currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
-	if query.Result != nil {
-		currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
-		if err != nil {
-			return ErrResp(http.StatusInternalServerError, err, "failed to load lastest configuration")
-		}
-		currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
-	}
-
-	// Copy the previously known secure settings
-	for i, r := range body.AlertmanagerConfig.Receivers {
-		for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
-			if gr.UID == "" { // new receiver
-				continue
-			}
-
-			cgmr, ok := currentReceiverMap[gr.UID]
-			if !ok {
-				// it tries to update a receiver that didn't previously exist
-				return ErrResp(http.StatusBadRequest, fmt.Errorf("unknown receiver: %s", gr.UID), "")
-			}
-
-			// frontend sends only the secure settings that have to be updated
-			// therefore we have to copy from the last configuration only those secure settings not included in the request
-			for key := range cgmr.SecureSettings {
-				_, ok := body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key]
-				if !ok {
-					decryptedValue, err := cgmr.GetDecryptedSecret(key)
-					if err != nil {
-						return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", key)
-					}
-
-					if body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
-						body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
-					}
-
-					body.AlertmanagerConfig.Receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
-				}
-			}
+	if err := srv.loadSecureSettings(c.OrgId, body.AlertmanagerConfig.Receivers); err != nil {
+		var unknownReceiverError UnknownReceiverError
+		if errors.As(err, &unknownReceiverError) {
+			return ErrResp(http.StatusBadRequest, err, "")
 		}
+		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
 	if err := body.ProcessConfig(); err != nil {
@@ -265,6 +302,130 @@ func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body ap
 }
 
 func (srv AlertmanagerSrv) RoutePostAMAlerts(c *models.ReqContext, body apimodels.PostableAlerts) response.Response {
-	// not implemented
 	return NotImplementedResp
 }
+
+func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body apimodels.TestReceiversConfigParams) response.Response {
+	if !c.HasUserRole(models.ROLE_EDITOR) {
+		return accessForbiddenResp()
+	}
+
+	if err := srv.loadSecureSettings(c.OrgId, body.Receivers); err != nil {
+		var unknownReceiverError UnknownReceiverError
+		if errors.As(err, &unknownReceiverError) {
+			return ErrResp(http.StatusBadRequest, err, "")
+		}
+		return ErrResp(http.StatusInternalServerError, err, "")
+	}
+
+	if err := body.ProcessConfig(); err != nil {
+		return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
+	}
+
+	ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+		c.Req.Context(),
+		c.Req.Request,
+		defaultTestReceiversTimeout,
+		maxTestReceiversTimeout)
+	if err != nil {
+		return ErrResp(http.StatusBadRequest, err, "")
+	}
+	defer cancelFunc()
+
+	result, err := srv.am.TestReceivers(ctx, body)
+	if err != nil {
+		if errors.Is(err, notifier.ErrNoReceivers) {
+			return response.Error(http.StatusBadRequest, "", err)
+		}
+		return response.Error(http.StatusInternalServerError, "", err)
+	}
+
+	return response.JSON(statusForTestReceivers(result.Receivers), newTestReceiversResult(result))
+}
+
+// contextWithTimeoutFromRequest returns a context with a deadline set from the
+// Request-Timeout header in the HTTP request. If the header is absent then the
+// context will use the default timeout. The timeout in the Request-Timeout
+// header cannot exceed the maximum timeout.
+func contextWithTimeoutFromRequest(ctx context.Context, r *http.Request, defaultTimeout, maxTimeout time.Duration) (context.Context, context.CancelFunc, error) {
+	timeout := defaultTimeout
+	if s := strings.TrimSpace(r.Header.Get("Request-Timeout")); s != "" {
+		// the timeout is measured in seconds
+		v, err := strconv.ParseInt(s, 10, 16)
+		if err != nil {
+			return nil, nil, err
+		}
+		if d := time.Duration(v) * time.Second; d < maxTimeout {
+			timeout = d
+		} else {
+			return nil, nil, fmt.Errorf("exceeded maximum timeout of %d seconds", maxTimeout)
+		}
+	}
+	ctx, cancelFunc := context.WithTimeout(ctx, timeout)
+	return ctx, cancelFunc, nil
+}
+
+func newTestReceiversResult(r *notifier.TestReceiversResult) apimodels.TestReceiversResult {
+	v := apimodels.TestReceiversResult{
+		Receivers: make([]apimodels.TestReceiverResult, len(r.Receivers)),
+		NotifedAt: r.NotifedAt,
+	}
+	for ix, next := range r.Receivers {
+		configs := make([]apimodels.TestReceiverConfigResult, len(next.Configs))
+		for jx, config := range next.Configs {
+			configs[jx].Name = config.Name
+			configs[jx].UID = config.UID
+			configs[jx].Status = config.Status
+			if config.Error != nil {
+				configs[jx].Error = config.Error.Error()
+			}
+		}
+		v.Receivers[ix].Configs = configs
+		v.Receivers[ix].Name = next.Name
+	}
+	return v
+}
+
+// statusForTestReceivers returns the appropriate status code for the response
+// for the results.
+//
+// It returns an HTTP 200 OK status code if notifications were sent to all receivers,
+// an HTTP 400 Bad Request status code if all receivers contain invalid configuration,
+// an HTTP 408 Request Timeout status code if all receivers timed out when sending
+// a test notification or an HTTP 207 Multi Status.
+func statusForTestReceivers(v []notifier.TestReceiverResult) int {
+	var (
+		numBadRequests   int
+		numTimeouts      int
+		numUnknownErrors int
+	)
+	for _, receiver := range v {
+		for _, next := range receiver.Configs {
+			if next.Error != nil {
+				var (
+					invalidReceiverErr notifier.InvalidReceiverError
+					receiverTimeoutErr notifier.ReceiverTimeoutError
+				)
+				if errors.As(next.Error, &invalidReceiverErr) {
+					numBadRequests += 1
+				} else if errors.As(next.Error, &receiverTimeoutErr) {
+					numTimeouts += 1
+				} else {
+					numUnknownErrors += 1
+				}
+			}
+		}
+	}
+	if numBadRequests == len(v) {
+		// if all receivers contain invalid configuration
+		return http.StatusBadRequest
+	} else if numTimeouts == len(v) {
+		// if all receivers contain valid configuration but timed out
+		return http.StatusRequestTimeout
+	} else if numBadRequests+numTimeouts+numUnknownErrors > 0 {
+		return http.StatusMultiStatus
+	} else {
+		// all receivers were sent a notification without error
+		return http.StatusOK
+	}
+}

pkg/services/ngalert/api/api_alertmanager_test.go

@@ -0,0 +1,140 @@
+package api
+
+import (
+	"context"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
+	"github.com/stretchr/testify/require"
+)
+
+func TestContextWithTimeoutFromRequest(t *testing.T) {
+	t.Run("assert context has default timeout when header is absent", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "https://grafana.net", nil)
+		require.NoError(t, err)
+
+		now := time.Now()
+		ctx := context.Background()
+		ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+			ctx,
+			req,
+			15*time.Second,
+			30*time.Second)
+		require.NoError(t, err)
+		require.NotNil(t, cancelFunc)
+		require.NotNil(t, ctx)
+
+		deadline, ok := ctx.Deadline()
+		require.True(t, ok)
+		require.True(t, deadline.After(now))
+		require.Less(t, deadline.Sub(now).Seconds(), 30.0)
+		require.GreaterOrEqual(t, deadline.Sub(now).Seconds(), 15.0)
+	})
+
+	t.Run("assert context has timeout in request header", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "https://grafana.net", nil)
+		require.NoError(t, err)
+		req.Header.Set("Request-Timeout", "5")
+
+		now := time.Now()
+		ctx := context.Background()
+		ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+			ctx,
+			req,
+			15*time.Second,
+			30*time.Second)
+		require.NoError(t, err)
+		require.NotNil(t, cancelFunc)
+		require.NotNil(t, ctx)
+
+		deadline, ok := ctx.Deadline()
+		require.True(t, ok)
+		require.True(t, deadline.After(now))
+		require.Less(t, deadline.Sub(now).Seconds(), 15.0)
+		require.GreaterOrEqual(t, deadline.Sub(now).Seconds(), 5.0)
+	})
+
+	t.Run("assert timeout in request header cannot exceed max timeout", func(t *testing.T) {
+		req, err := http.NewRequest(http.MethodGet, "https://grafana.net", nil)
+		require.NoError(t, err)
+		req.Header.Set("Request-Timeout", "60")
+
+		ctx := context.Background()
+		ctx, cancelFunc, err := contextWithTimeoutFromRequest(
+			ctx,
+			req,
+			15*time.Second,
+			30*time.Second)
+		require.Error(t, err, "exceeded maximum timeout")
+		require.Nil(t, cancelFunc)
+		require.Nil(t, ctx)
+	})
+}
+
+func TestStatusForTestReceivers(t *testing.T) {
+	t.Run("assert HTTP 400 Status Bad Request for no receivers", func(t *testing.T) {
+		require.Equal(t, http.StatusBadRequest, statusForTestReceivers([]notifier.TestReceiverResult{}))
+	})
+
+	t.Run("assert HTTP 400 Bad Request when all invalid receivers", func(t *testing.T) {
+		require.Equal(t, http.StatusBadRequest, statusForTestReceivers([]notifier.TestReceiverResult{{
+			Name: "test1",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test1",
+				UID:    "uid1",
+				Status: "failed",
+				Error:  notifier.InvalidReceiverError{},
+			}},
+		}, {
+			Name: "test2",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test2",
+				UID:    "uid2",
+				Status: "failed",
+				Error:  notifier.InvalidReceiverError{},
+			}},
+		}}))
+	})
+
+	t.Run("assert HTTP 408 Request Timeout when all receivers timed out", func(t *testing.T) {
+		require.Equal(t, http.StatusRequestTimeout, statusForTestReceivers([]notifier.TestReceiverResult{{
+			Name: "test1",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test1",
+				UID:    "uid1",
+				Status: "failed",
+				Error:  notifier.ReceiverTimeoutError{},
+			}},
+		}, {
+			Name: "test2",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test2",
+				UID:    "uid2",
+				Status: "failed",
+				Error:  notifier.ReceiverTimeoutError{},
+			}},
+		}}))
+	})
+
+	t.Run("assert 207 Multi Status for different errors", func(t *testing.T) {
+		require.Equal(t, http.StatusMultiStatus, statusForTestReceivers([]notifier.TestReceiverResult{{
+			Name: "test1",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test1",
+				UID:    "uid1",
+				Status: "failed",
+				Error:  notifier.InvalidReceiverError{},
+			}},
+		}, {
+			Name: "test2",
+			Configs: []notifier.TestReceiverConfigResult{{
+				Name:   "test2",
+				UID:    "uid2",
+				Status: "failed",
+				Error:  notifier.ReceiverTimeoutError{},
+			}},
+		}}))
+	})
+}

pkg/services/ngalert/api/forked_am.go

@@ -146,3 +146,12 @@ func (am *ForkedAMSvc) RoutePostAMAlerts(ctx *models.ReqContext, body apimodels.
 
 	return s.RoutePostAMAlerts(ctx, body)
 }
+
+func (am *ForkedAMSvc) RoutePostTestReceivers(ctx *models.ReqContext, body apimodels.TestReceiversConfigParams) response.Response {
+	s, err := am.getService(ctx)
+	if err != nil {
+		return ErrResp(400, err, "")
+	}
+
+	return s.RoutePostTestReceivers(ctx, body)
+}

pkg/services/ngalert/api/generated_base_api_alertmanager.go

@@ -31,6 +31,7 @@ type AlertmanagerApiService interface {
 	RouteGetSilences(*models.ReqContext) response.Response
 	RoutePostAMAlerts(*models.ReqContext, apimodels.PostableAlerts) response.Response
 	RoutePostAlertingConfig(*models.ReqContext, apimodels.PostableUserConfig) response.Response
+	RoutePostTestReceivers(*models.ReqContext, apimodels.TestReceiversConfigParams) response.Response
 }
 
 func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService, m *metrics.Metrics) {
@@ -137,5 +138,15 @@ func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService, m *
 				m,
 			),
 		)
+		group.Post(
+			toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/receivers/test"),
+			binding.Bind(apimodels.TestReceiversConfigParams{}),
+			metrics.Instrument(
+				http.MethodPost,
+				"/api/alertmanager/{Recipient}/config/api/v1/receivers/test",
+				srv.RoutePostTestReceivers,
+				m,
+			),
+		)
 	}, middleware.ReqSignedIn)
 }

pkg/services/ngalert/api/lotex_am.go

@@ -192,3 +192,7 @@ func (am *LotexAM) RoutePostAMAlerts(ctx *models.ReqContext, alerts apimodels.Po
 		nil,
 	)
 }
+
+func (am *LotexAM) RoutePostTestReceivers(ctx *models.ReqContext, config apimodels.TestReceiversConfigParams) response.Response {
+	return NotImplementedResp
+}

pkg/services/ngalert/api/tooling/definitions/alertmanager.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"time"
 
 	"github.com/go-openapi/strfmt"
 	"github.com/pkg/errors"
@@ -73,6 +74,17 @@ import (
 //       200: alertGroups
 //       400: ValidationError
 
+// swagger:route POST /api/alertmanager/{Recipient}/config/api/v1/receivers/test alertmanager RoutePostTestReceivers
+//
+// Test Grafana managed receivers without saving them.
+//
+//     Responses:
+//
+//       200: Ack
+//       207: MultiStatus
+//       400: ValidationError
+//       408: Failure
+
 // swagger:route GET /api/alertmanager/{Recipient}/api/v2/silences alertmanager RouteGetSilences
 //
 // get silences
@@ -105,6 +117,40 @@ import (
 //       200: Ack
 //       400: ValidationError
 
+// swagger:model
+type TestReceiversConfig struct {
+	Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"`
+}
+
+// swagger:parameters RoutePostTestReceivers
+type TestReceiversConfigParams struct {
+	Receivers []*PostableApiReceiver `yaml:"receivers,omitempty" json:"receivers,omitempty"`
+}
+
+func (c *TestReceiversConfigParams) ProcessConfig() error {
+	return processReceiverConfigs(c.Receivers)
+}
+
+// swagger:model
+type TestReceiversResult struct {
+	Receivers []TestReceiverResult `json:"receivers"`
+	NotifedAt time.Time            `json:"notified_at"`
+}
+
+// swagger:model
+type TestReceiverResult struct {
+	Name    string                     `json:"name"`
+	Configs []TestReceiverConfigResult `json:"grafana_managed_receiver_configs"`
+}
+
+// swagger:model
+type TestReceiverConfigResult struct {
+	Name   string `json:"name"`
+	UID    string `json:"uid"`
+	Status string `json:"status"`
+	Error  string `json:"error,omitempty"`
+}
+
 // swagger:parameters RouteCreateSilence
 type CreateSilenceParams struct {
 	// in:body
@@ -345,39 +391,7 @@ func (c *PostableUserConfig) GetGrafanaReceiverMap() map[string]*PostableGrafana
 
 // ProcessConfig parses grafana receivers, encrypts secrets and assigns UUIDs (if they are missing)
 func (c *PostableUserConfig) ProcessConfig() error {
-	seenUIDs := make(map[string]struct{})
-	// encrypt secure settings for storing them in DB
-	for _, r := range c.AlertmanagerConfig.Receivers {
-		switch r.Type() {
-		case GrafanaReceiverType:
-			for _, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
-				for k, v := range gr.SecureSettings {
-					encryptedData, err := util.Encrypt([]byte(v), setting.SecretKey)
-					if err != nil {
-						return fmt.Errorf("failed to encrypt secure settings: %w", err)
-					}
-					gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(encryptedData)
-				}
-				if gr.UID == "" {
-					retries := 5
-					for i := 0; i < retries; i++ {
-						gen := util.GenerateShortUID()
-						_, ok := seenUIDs[gen]
-						if !ok {
-							gr.UID = gen
-							break
-						}
-					}
-					if gr.UID == "" {
-						return fmt.Errorf("all %d attempts to generate UID for receiver have failed; please retry", retries)
-					}
-				}
-				seenUIDs[gr.UID] = struct{}{}
-			}
-		default:
-		}
-	}
-	return nil
+	return processReceiverConfigs(c.AlertmanagerConfig.Receivers)
 }
 
 // MarshalYAML implements yaml.Marshaller.
@@ -911,3 +925,39 @@ type GettableGrafanaReceivers struct {
 type PostableGrafanaReceivers struct {
 	GrafanaManagedReceivers []*PostableGrafanaReceiver `yaml:"grafana_managed_receiver_configs,omitempty" json:"grafana_managed_receiver_configs,omitempty"`
 }
+
+func processReceiverConfigs(c []*PostableApiReceiver) error {
+	seenUIDs := make(map[string]struct{})
+	// encrypt secure settings for storing them in DB
+	for _, r := range c {
+		switch r.Type() {
+		case GrafanaReceiverType:
+			for _, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
+				for k, v := range gr.SecureSettings {
+					encryptedData, err := util.Encrypt([]byte(v), setting.SecretKey)
+					if err != nil {
+						return fmt.Errorf("failed to encrypt secure settings: %w", err)
+					}
+					gr.SecureSettings[k] = base64.StdEncoding.EncodeToString(encryptedData)
+				}
+				if gr.UID == "" {
+					retries := 5
+					for i := 0; i < retries; i++ {
+						gen := util.GenerateShortUID()
+						_, ok := seenUIDs[gen]
+						if !ok {
+							gr.UID = gen
+							break
+						}
+					}
+					if gr.UID == "" {
+						return fmt.Errorf("all %d attempts to generate UID for receiver have failed; please retry", retries)
+					}
+				}
+				seenUIDs[gr.UID] = struct{}{}
+			}
+		default:
+		}
+	}
+	return nil
+}

pkg/services/ngalert/api/tooling/spec.json

@@ -485,6 +485,49 @@
         }
       }
     },
+    "/api/alertmanager/{Recipient}/config/api/v1/receivers/test": {
+      "post": {
+        "tags": [
+          "alertmanager"
+        ],
+        "summary": "Test Grafana managed receivers without saving them.",
+        "operationId": "RoutePostTestReceivers",
+        "parameters": [
+          {
+            "type": "array",
+            "items": {
+              "$ref": "#/definitions/PostableApiReceiver"
+            },
+            "x-go-name": "Receivers",
+            "name": "receivers",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Ack",
+            "schema": {
+              "$ref": "#/definitions/Ack"
+            }
+          },
+          "207": {
+            "$ref": "#/responses/MultiStatus"
+          },
+          "400": {
+            "description": "ValidationError",
+            "schema": {
+              "$ref": "#/definitions/ValidationError"
+            }
+          },
+          "408": {
+            "description": "Failure",
+            "schema": {
+              "$ref": "#/definitions/Failure"
+            }
+          }
+        }
+      }
+    },
     "/api/prometheus/{Recipient}/api/v1/alerts": {
       "get": {
         "description": "gets the current alerts",
@@ -1707,6 +1750,7 @@
           "enum": [
             "Alerting"
           ],
+          "x-go-enum-desc": "Alerting AlertingErrState",
           "x-go-name": "ExecErrState"
         },
         "id": {
@@ -1735,6 +1779,7 @@
             "NoData",
             "OK"
           ],
+          "x-go-enum-desc": "Alerting Alerting\nNoData NoData\nOK OK",
           "x-go-name": "NoDataState"
         },
         "orgId": {
@@ -2547,6 +2592,7 @@
           "enum": [
             "Alerting"
           ],
+          "x-go-enum-desc": "Alerting AlertingErrState",
           "x-go-name": "ExecErrState"
         },
         "no_data_state": {
@@ -2556,6 +2602,7 @@
             "NoData",
             "OK"
           ],
+          "x-go-enum-desc": "Alerting Alerting\nNoData NoData\nOK OK",
           "x-go-name": "NoDataState"
         },
         "title": {
@@ -3229,6 +3276,76 @@
       },
       "x-go-package": "github.com/prometheus/common/config"
     },
+    "TestReceiverConfigResult": {
+      "type": "object",
+      "properties": {
+        "error": {
+          "type": "string",
+          "x-go-name": "Error"
+        },
+        "name": {
+          "type": "string",
+          "x-go-name": "Name"
+        },
+        "status": {
+          "type": "string",
+          "x-go-name": "Status"
+        },
+        "uid": {
+          "type": "string",
+          "x-go-name": "UID"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
+    "TestReceiverResult": {
+      "type": "object",
+      "properties": {
+        "grafana_managed_receiver_configs": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/TestReceiverConfigResult"
+          },
+          "x-go-name": "Configs"
+        },
+        "name": {
+          "type": "string",
+          "x-go-name": "Name"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
+    "TestReceiversConfig": {
+      "type": "object",
+      "properties": {
+        "receivers": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/PostableApiReceiver"
+          },
+          "x-go-name": "Receivers"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
+    "TestReceiversResult": {
+      "type": "object",
+      "properties": {
+        "notified_at": {
+          "type": "string",
+          "format": "date-time",
+          "x-go-name": "NotifedAt"
+        },
+        "receivers": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/TestReceiverResult"
+          },
+          "x-go-name": "Receivers"
+        }
+      },
+      "x-go-package": "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+    },
     "TestRulePayload": {
       "type": "object",
       "properties": {
@@ -3483,11 +3600,12 @@
       "$ref": "#/definitions/alertGroup"
     },
     "alertGroups": {
-      "description": "AlertGroups alert groups",
       "type": "array",
       "items": {
         "$ref": "#/definitions/alertGroup"
       },
+      "x-go-name": "AlertGroups",
+      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/alertGroups"
     },
     "alertStatus": {
@@ -3672,16 +3790,14 @@
       "$ref": "#/definitions/gettableAlert"
     },
     "gettableAlerts": {
+      "description": "GettableAlerts gettable alerts",
       "type": "array",
       "items": {
         "$ref": "#/definitions/gettableAlert"
       },
-      "x-go-name": "GettableAlerts",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/gettableAlerts"
     },
     "gettableSilence": {
-      "description": "GettableSilence gettable silence",
       "type": "object",
       "required": [
         "comment",
@@ -3734,6 +3850,8 @@
           "x-go-name": "UpdatedAt"
         }
       },
+      "x-go-name": "GettableSilence",
+      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/gettableSilence"
     },
     "gettableSilences": {
@@ -3872,6 +3990,7 @@
       "x-go-package": "github.com/prometheus/alertmanager/api/v2/models"
     },
     "postableSilence": {
+      "description": "PostableSilence postable silence",
       "type": "object",
       "required": [
         "comment",
@@ -3912,8 +4031,6 @@
           "x-go-name": "StartsAt"
         }
       },
-      "x-go-name": "PostableSilence",
-      "x-go-package": "github.com/prometheus/alertmanager/api/v2/models",
       "$ref": "#/definitions/postableSilence"
     },
     "receiver": {

pkg/services/ngalert/notifier/alertmanager.go

@@ -106,7 +106,8 @@ type Alertmanager struct {
 	dispatcherMetrics *dispatch.DispatcherMetrics
 
 	reloadConfigMtx sync.RWMutex
-	config          []byte
+	config          *apimodels.PostableUserConfig
+	configHash      [16]byte
 }
 
 func New(cfg *setting.Cfg, store store.AlertingStore, m *metrics.Metrics) (*Alertmanager, error) {
@@ -166,7 +167,11 @@ func (am *Alertmanager) Ready() bool {
 	am.reloadConfigMtx.RLock()
 	defer am.reloadConfigMtx.RUnlock()
 
-	return len(am.config) > 0
+	return am.ready()
+}
+
+func (am *Alertmanager) ready() bool {
+	return am.config != nil
 }
 
 func (am *Alertmanager) Run(ctx context.Context) error {
@@ -314,6 +319,32 @@ func (am *Alertmanager) SyncAndApplyConfigFromDatabase(orgID int64) error {
 	return nil
 }
 
+func (am *Alertmanager) getTemplate() (*template.Template, error) {
+	am.reloadConfigMtx.RLock()
+	defer am.reloadConfigMtx.RUnlock()
+	if !am.ready() {
+		return nil, errors.New("alertmanager is not initialized")
+	}
+	paths := make([]string, 0, len(am.config.TemplateFiles))
+	for name := range am.config.TemplateFiles {
+		paths = append(paths, filepath.Join(am.WorkingDirPath(), name))
+	}
+	return am.templateFromPaths(paths...)
+}
+
+func (am *Alertmanager) templateFromPaths(paths ...string) (*template.Template, error) {
+	tmpl, err := template.FromGlobs(paths...)
+	if err != nil {
+		return nil, err
+	}
+	externalURL, err := url.Parse(am.Settings.AppURL)
+	if err != nil {
+		return nil, err
+	}
+	tmpl.ExternalURL = externalURL
+	return tmpl, nil
+}
+
 // applyConfig applies a new configuration by re-initializing all components using the configuration provided.
 // It is not safe to call concurrently.
 func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig []byte) (err error) {
@@ -328,7 +359,7 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig
 		rawConfig = enc
 	}
 
-	if md5.Sum(am.config) != md5.Sum(rawConfig) {
+	if am.configHash != md5.Sum(rawConfig) {
 		configChanged = true
 	}
 
@@ -350,15 +381,10 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig
 	}
 
 	// With the templates persisted, create the template list using the paths.
-	tmpl, err := template.FromGlobs(paths...)
+	tmpl, err := am.templateFromPaths(paths...)
 	if err != nil {
 		return err
 	}
-	externalURL, err := url.Parse(am.Settings.AppURL)
-	if err != nil {
-		return err
-	}
-	tmpl.ExternalURL = externalURL
 
 	// Finally, build the integrations map using the receiver configuration and templates.
 	integrationsMap, err := am.buildIntegrationsMap(cfg.AlertmanagerConfig.Receivers, tmpl)
@@ -400,7 +426,9 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig
 		am.inhibitor.Run()
 	}()
 
-	am.config = rawConfig
+	am.config = cfg
+	am.configHash = md5.Sum(rawConfig)
+
 	return nil
 }
 
@@ -430,77 +458,95 @@ type NotificationChannel interface {
 // buildReceiverIntegrations builds a list of integration notifiers off of a receiver config.
 func (am *Alertmanager) buildReceiverIntegrations(receiver *apimodels.PostableApiReceiver, tmpl *template.Template) ([]notify.Integration, error) {
 	var integrations []notify.Integration
-
 	for i, r := range receiver.GrafanaManagedReceivers {
-		// secure settings are already encrypted at this point
-		secureSettings := securejsondata.SecureJsonData(make(map[string][]byte, len(r.SecureSettings)))
-
-		for k, v := range r.SecureSettings {
-			d, err := base64.StdEncoding.DecodeString(v)
-			if err != nil {
-				return nil, fmt.Errorf("failed to decode secure setting")
-			}
-			secureSettings[k] = d
-		}
-		var (
-			cfg = &channels.NotificationChannelConfig{
-				UID:                   r.UID,
-				Name:                  r.Name,
-				Type:                  r.Type,
-				DisableResolveMessage: r.DisableResolveMessage,
-				Settings:              r.Settings,
-				SecureSettings:        secureSettings,
-			}
-			n   NotificationChannel
-			err error
-		)
-		switch r.Type {
-		case "email":
-			n, err = channels.NewEmailNotifier(cfg, tmpl) // Email notifier already has a default template.
-		case "pagerduty":
-			n, err = channels.NewPagerdutyNotifier(cfg, tmpl)
-		case "pushover":
-			n, err = channels.NewPushoverNotifier(cfg, tmpl)
-		case "slack":
-			n, err = channels.NewSlackNotifier(cfg, tmpl)
-		case "telegram":
-			n, err = channels.NewTelegramNotifier(cfg, tmpl)
-		case "victorops":
-			n, err = channels.NewVictoropsNotifier(cfg, tmpl)
-		case "teams":
-			n, err = channels.NewTeamsNotifier(cfg, tmpl)
-		case "dingding":
-			n, err = channels.NewDingDingNotifier(cfg, tmpl)
-		case "kafka":
-			n, err = channels.NewKafkaNotifier(cfg, tmpl)
-		case "webhook":
-			n, err = channels.NewWebHookNotifier(cfg, tmpl)
-		case "sensugo":
-			n, err = channels.NewSensuGoNotifier(cfg, tmpl)
-		case "discord":
-			n, err = channels.NewDiscordNotifier(cfg, tmpl)
-		case "googlechat":
-			n, err = channels.NewGoogleChatNotifier(cfg, tmpl)
-		case "LINE":
-			n, err = channels.NewLineNotifier(cfg, tmpl)
-		case "threema":
-			n, err = channels.NewThreemaNotifier(cfg, tmpl)
-		case "opsgenie":
-			n, err = channels.NewOpsgenieNotifier(cfg, tmpl)
-		case "prometheus-alertmanager":
-			n, err = channels.NewAlertmanagerNotifier(cfg, tmpl)
-		default:
-			return nil, fmt.Errorf("notifier %s is not supported", r.Type)
-		}
+		n, err := am.buildReceiverIntegration(r, tmpl)
 		if err != nil {
 			return nil, err
 		}
 		integrations = append(integrations, notify.NewIntegration(n, n, r.Type, i))
 	}
-
 	return integrations, nil
 }
 
+func (am *Alertmanager) buildReceiverIntegration(r *apimodels.PostableGrafanaReceiver, tmpl *template.Template) (NotificationChannel, error) {
+	// secure settings are already encrypted at this point
+	secureSettings := securejsondata.SecureJsonData(make(map[string][]byte, len(r.SecureSettings)))
+
+	for k, v := range r.SecureSettings {
+		d, err := base64.StdEncoding.DecodeString(v)
+		if err != nil {
+			return nil, InvalidReceiverError{
+				Receiver: r,
+				Err:      errors.New("failed to decode secure setting"),
+			}
+		}
+		secureSettings[k] = d
+	}
+
+	var (
+		cfg = &channels.NotificationChannelConfig{
+			UID:                   r.UID,
+			Name:                  r.Name,
+			Type:                  r.Type,
+			DisableResolveMessage: r.DisableResolveMessage,
+			Settings:              r.Settings,
+			SecureSettings:        secureSettings,
+		}
+		n   NotificationChannel
+		err error
+	)
+	switch r.Type {
+	case "email":
+		n, err = channels.NewEmailNotifier(cfg, tmpl) // Email notifier already has a default template.
+	case "pagerduty":
+		n, err = channels.NewPagerdutyNotifier(cfg, tmpl)
+	case "pushover":
+		n, err = channels.NewPushoverNotifier(cfg, tmpl)
+	case "slack":
+		n, err = channels.NewSlackNotifier(cfg, tmpl)
+	case "telegram":
+		n, err = channels.NewTelegramNotifier(cfg, tmpl)
+	case "victorops":
+		n, err = channels.NewVictoropsNotifier(cfg, tmpl)
+	case "teams":
+		n, err = channels.NewTeamsNotifier(cfg, tmpl)
+	case "dingding":
+		n, err = channels.NewDingDingNotifier(cfg, tmpl)
+	case "kafka":
+		n, err = channels.NewKafkaNotifier(cfg, tmpl)
+	case "webhook":
+		n, err = channels.NewWebHookNotifier(cfg, tmpl)
+	case "sensugo":
+		n, err = channels.NewSensuGoNotifier(cfg, tmpl)
+	case "discord":
+		n, err = channels.NewDiscordNotifier(cfg, tmpl)
+	case "googlechat":
+		n, err = channels.NewGoogleChatNotifier(cfg, tmpl)
+	case "LINE":
+		n, err = channels.NewLineNotifier(cfg, tmpl)
+	case "threema":
+		n, err = channels.NewThreemaNotifier(cfg, tmpl)
+	case "opsgenie":
+		n, err = channels.NewOpsgenieNotifier(cfg, tmpl)
+	case "prometheus-alertmanager":
+		n, err = channels.NewAlertmanagerNotifier(cfg, tmpl)
+	default:
+		return nil, InvalidReceiverError{
+			Receiver: r,
+			Err:      fmt.Errorf("notifier %s is not supported", r.Type),
+		}
+	}
+
+	if err != nil {
+		return nil, InvalidReceiverError{
+			Receiver: r,
+			Err:      err,
+		}
+	}
+
+	return n, nil
+}
+
 // PutAlerts receives the alerts and then sends them through the corresponding route based on whenever the alert has a receiver embedded or not
 func (am *Alertmanager) PutAlerts(postableAlerts apimodels.PostableAlerts) error {
 	now := time.Now()

pkg/services/ngalert/notifier/channels/webhook.go

@@ -31,6 +31,9 @@ type WebhookNotifier struct {
 // NewWebHookNotifier is the constructor for
 // the WebHook notifier.
 func NewWebHookNotifier(model *NotificationChannelConfig, t *template.Template) (*WebhookNotifier, error) {
+	if model.Settings == nil {
+		return nil, receiverInitError{Cfg: *model, Reason: "could not find settings property"}
+	}
 	url := model.Settings.Get("url").MustString()
 	if url == "" {
 		return nil, receiverInitError{Cfg: *model, Reason: "could not find url property in settings"}

pkg/services/ngalert/notifier/receivers.go

@@ -0,0 +1,227 @@
+package notifier
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"time"
+
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/prometheus/alertmanager/notify"
+	"github.com/prometheus/alertmanager/types"
+	"github.com/prometheus/common/model"
+	"golang.org/x/sync/errgroup"
+)
+
+const (
+	maxTestReceiversWorkers = 10
+)
+
+var (
+	ErrNoReceivers = errors.New("no receivers")
+)
+
+type TestReceiversResult struct {
+	Receivers []TestReceiverResult
+	NotifedAt time.Time
+}
+
+type TestReceiverResult struct {
+	Name    string
+	Configs []TestReceiverConfigResult
+}
+
+type TestReceiverConfigResult struct {
+	Name   string
+	UID    string
+	Status string
+	Error  error
+}
+
+type InvalidReceiverError struct {
+	Receiver *apimodels.PostableGrafanaReceiver
+	Err      error
+}
+
+func (e InvalidReceiverError) Error() string {
+	return fmt.Sprintf("the receiver is invalid: %s", e.Err)
+}
+
+type ReceiverTimeoutError struct {
+	Receiver *apimodels.PostableGrafanaReceiver
+	Err      error
+}
+
+func (e ReceiverTimeoutError) Error() string {
+	return fmt.Sprintf("the receiver timed out: %s", e.Err)
+}
+
+func (am *Alertmanager) TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigParams) (*TestReceiversResult, error) {
+	// now represents the start time of the test
+	now := time.Now()
+	testAlert := &types.Alert{
+		Alert: model.Alert{
+			Labels: model.LabelSet{
+				model.LabelName("alertname"): "TestAlertAlwaysFiring",
+				model.LabelName("instance"):  "Grafana",
+			},
+			Annotations: model.LabelSet{
+				model.LabelName("summary"):     "TestAlertAlwaysFiring",
+				model.LabelName("description"): "This is a test alert from Grafana",
+			},
+			StartsAt: now,
+		},
+		UpdatedAt: now,
+	}
+
+	// we must set a group key that is unique per test as some receivers use this key to deduplicate alerts
+	ctx = notify.WithGroupKey(ctx, testAlert.Labels.String()+now.String())
+
+	tmpl, err := am.getTemplate()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get template: %w", err)
+	}
+
+	// job contains all metadata required to test a receiver
+	type job struct {
+		Config       *apimodels.PostableGrafanaReceiver
+		ReceiverName string
+		Notifier     notify.Notifier
+	}
+
+	// result contains the receiver that was tested and an error that is non-nil if the test failed
+	type result struct {
+		Config       *apimodels.PostableGrafanaReceiver
+		ReceiverName string
+		Error        error
+	}
+
+	newTestReceiversResult := func(results []result, notifiedAt time.Time) *TestReceiversResult {
+		m := make(map[string]TestReceiverResult)
+		for _, receiver := range c.Receivers {
+			// set up the result for this receiver
+			m[receiver.Name] = TestReceiverResult{
+				Name: receiver.Name,
+				// A Grafana receiver can have multiple nested receivers
+				Configs: make([]TestReceiverConfigResult, 0, len(receiver.GrafanaManagedReceivers)),
+			}
+		}
+		for _, next := range results {
+			tmp := m[next.ReceiverName]
+			status := "ok"
+			if next.Error != nil {
+				status = "failed"
+			}
+			tmp.Configs = append(tmp.Configs, TestReceiverConfigResult{
+				Name:   next.Config.Name,
+				UID:    next.Config.UID,
+				Status: status,
+				Error:  processNotifierError(next.Config, next.Error),
+			})
+			m[next.ReceiverName] = tmp
+		}
+		v := new(TestReceiversResult)
+		v.Receivers = make([]TestReceiverResult, 0, len(c.Receivers))
+		v.NotifedAt = notifiedAt
+		for _, next := range m {
+			v.Receivers = append(v.Receivers, next)
+		}
+		return v
+	}
+
+	// invalid keeps track of all invalid receiver configurations
+	invalid := make([]result, 0, len(c.Receivers))
+	// jobs keeps track of all receivers that need to be sent test notifications
+	jobs := make([]job, 0, len(c.Receivers))
+
+	for _, receiver := range c.Receivers {
+		for _, next := range receiver.GrafanaManagedReceivers {
+			n, err := am.buildReceiverIntegration(next, tmpl)
+			if err != nil {
+				invalid = append(invalid, result{
+					Config:       next,
+					ReceiverName: next.Name,
+					Error:        err,
+				})
+			} else {
+				jobs = append(jobs, job{
+					Config:       next,
+					ReceiverName: receiver.Name,
+					Notifier:     n,
+				})
+			}
+		}
+	}
+
+	if len(invalid)+len(jobs) == 0 {
+		return nil, ErrNoReceivers
+	}
+
+	if len(jobs) == 0 {
+		return newTestReceiversResult(invalid, now), nil
+	}
+
+	numWorkers := maxTestReceiversWorkers
+	if numWorkers > len(jobs) {
+		numWorkers = len(jobs)
+	}
+
+	resultCh := make(chan result, len(jobs))
+	workCh := make(chan job, len(jobs))
+	for _, job := range jobs {
+		workCh <- job
+	}
+	close(workCh)
+
+	g, ctx := errgroup.WithContext(ctx)
+	for i := 0; i < numWorkers; i++ {
+		g.Go(func() error {
+			for next := range workCh {
+				v := result{
+					Config:       next.Config,
+					ReceiverName: next.ReceiverName,
+				}
+				if _, err := next.Notifier.Notify(ctx, testAlert); err != nil {
+					v.Error = err
+				}
+				resultCh <- v
+			}
+			return nil
+		})
+	}
+	g.Wait() // nolint
+	close(resultCh)
+
+	results := make([]result, 0, len(jobs))
+	for next := range resultCh {
+		results = append(results, next)
+	}
+
+	return newTestReceiversResult(append(invalid, results...), now), nil
+}
+
+func processNotifierError(config *apimodels.PostableGrafanaReceiver, err error) error {
+	if err == nil {
+		return nil
+	}
+
+	var urlError *url.Error
+	if errors.As(err, &urlError) {
+		if urlError.Timeout() {
+			return ReceiverTimeoutError{
+				Receiver: config,
+				Err:      err,
+			}
+		}
+	}
+
+	if errors.Is(err, context.DeadlineExceeded) {
+		return ReceiverTimeoutError{
+			Receiver: config,
+			Err:      err,
+		}
+	}
+
+	return err
+}

pkg/services/ngalert/notifier/receivers_test.go

@@ -0,0 +1,82 @@
+package notifier
+
+import (
+	"context"
+	"errors"
+	"net/url"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+)
+
+func TestInvalidReceiverError_Error(t *testing.T) {
+	e := InvalidReceiverError{
+		Receiver: &definitions.PostableGrafanaReceiver{
+			Name: "test",
+			UID:  "uid",
+		},
+		Err: errors.New("this is an error"),
+	}
+	require.Equal(t, "the receiver is invalid: this is an error", e.Error())
+}
+
+func TestReceiverTimeoutError_Error(t *testing.T) {
+	e := ReceiverTimeoutError{
+		Receiver: &definitions.PostableGrafanaReceiver{
+			Name: "test",
+			UID:  "uid",
+		},
+		Err: errors.New("context deadline exceeded"),
+	}
+	require.Equal(t, "the receiver timed out: context deadline exceeded", e.Error())
+}
+
+type timeoutError struct{}
+
+func (e timeoutError) Error() string {
+	return "the request timed out"
+}
+
+func (e timeoutError) Timeout() bool {
+	return true
+}
+
+func TestProcessNotifierError(t *testing.T) {
+	t.Run("assert ReceiverTimeoutError is returned for context deadline exceeded", func(t *testing.T) {
+		r := &definitions.PostableGrafanaReceiver{
+			Name: "test",
+			UID:  "uid",
+		}
+		require.Equal(t, ReceiverTimeoutError{
+			Receiver: r,
+			Err:      context.DeadlineExceeded,
+		}, processNotifierError(r, context.DeadlineExceeded))
+	})
+
+	t.Run("assert ReceiverTimeoutError is returned for *url.Error timeout", func(t *testing.T) {
+		r := &definitions.PostableGrafanaReceiver{
+			Name: "test",
+			UID:  "uid",
+		}
+		urlError := &url.Error{
+			Op:  "Get",
+			URL: "https://grafana.net",
+			Err: timeoutError{},
+		}
+		require.Equal(t, ReceiverTimeoutError{
+			Receiver: r,
+			Err:      urlError,
+		}, processNotifierError(r, urlError))
+	})
+
+	t.Run("assert unknown error is returned unmodified", func(t *testing.T) {
+		r := &definitions.PostableGrafanaReceiver{
+			Name: "test",
+			UID:  "uid",
+		}
+		err := errors.New("this is an error")
+		require.Equal(t, err, processNotifierError(r, err))
+	})
+}

pkg/services/ngalert/notifier/status.go

@@ -1,8 +1,6 @@
 package notifier
 
 import (
-	"encoding/json"
-
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 )
 
@@ -10,13 +8,9 @@ func (am *Alertmanager) GetStatus() apimodels.GettableStatus {
 	am.reloadConfigMtx.RLock()
 	defer am.reloadConfigMtx.RUnlock()
 
-	var amConfig apimodels.PostableApiAlertingConfig
-	if am.config != nil {
-		err := json.Unmarshal(am.config, &amConfig)
-		if err != nil {
-			// this should never error here, if the configuration is running it should be valid.
-			am.logger.Error("unable to marshal alertmanager configuration", "err", err)
-		}
+	config := apimodels.PostableApiAlertingConfig{}
+	if am.ready() {
+		config = am.config.AlertmanagerConfig
 	}
-	return *apimodels.NewGettableStatus(&amConfig)
+	return *apimodels.NewGettableStatus(&config)
 }

pkg/tests/api/alerting/api_alertmanager_configuration_test.go

@@ -85,7 +85,7 @@ func TestAlertmanagerConfigurationIsTransactional(t *testing.T) {
 }
 `
 		resp := postRequest(t, alertConfigURL, payload, http.StatusBadRequest) // nolint
-		require.JSONEq(t, `{"message":"failed to save and apply Alertmanager configuration: failed to validate receiver \"slack.receiver\" of type \"slack\": token must be specified when using the Slack chat API"}`, getBody(t, resp.Body))
+		require.JSONEq(t, `{"message":"failed to save and apply Alertmanager configuration: the receiver is invalid: failed to validate receiver \"slack.receiver\" of type \"slack\": token must be specified when using the Slack chat API"}`, getBody(t, resp.Body))
 
 		resp = getRequest(t, alertConfigURL, http.StatusOK) // nolint
 		require.JSONEq(t, defaultAlertmanagerConfigJSON, getBody(t, resp.Body))

pkg/tests/api/alerting/api_notification_channel_test.go

@@ -30,6 +30,328 @@ import (
 	"github.com/grafana/grafana/pkg/tests/testinfra"
 )
 
+func TestTestReceivers(t *testing.T) {
+	t.Run("assert no receivers returns 400 Bad Request", func(t *testing.T) {
+		// Setup Grafana and its Database
+		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+			EnableFeatureToggles: []string{"ngalert"},
+		})
+		store := testinfra.SetUpDatabase(t, dir)
+		store.Bus = bus.GetBus()
+		grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
+		createUser(t, store, models.CreateUserCommand{
+			DefaultOrgRole: string(models.ROLE_EDITOR),
+			Login:          "grafana",
+			Password:       "password",
+		})
+
+		testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
+		// nolint
+		resp := postRequest(t, testReceiversURL, `{
+	"receivers": []
+}`, http.StatusBadRequest)
+		t.Cleanup(func() {
+			err := resp.Body.Close()
+			require.NoError(t, err)
+		})
+
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.JSONEq(t, `{"error":"no receivers"}`, string(b))
+	})
+
+	t.Run("assert working receiver returns OK", func(t *testing.T) {
+		// Setup Grafana and its Database
+		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+			EnableFeatureToggles: []string{"ngalert"},
+		})
+		store := testinfra.SetUpDatabase(t, dir)
+		store.Bus = bus.GetBus()
+		grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
+		createUser(t, store, models.CreateUserCommand{
+			DefaultOrgRole: string(models.ROLE_EDITOR),
+			Login:          "grafana",
+			Password:       "password",
+		})
+
+		oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync")
+		mockEmails := &mockEmailHandler{}
+		bus.AddHandlerCtx("", mockEmails.sendEmailCommandHandlerSync)
+		t.Cleanup(func() {
+			bus.AddHandlerCtx("", oldEmailBus)
+		})
+
+		testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
+		// nolint
+		resp := postRequest(t, testReceiversURL, `{
+	"receivers": [{
+		"name":"receiver-1",
+		"grafana_managed_receiver_configs": [
+			{
+				"uid":"",
+				"name":"receiver-1",
+				"type":"email",
+				"disableResolveMessage":false,
+				"settings":{
+					"addresses":"example@email.com"
+				},
+				"secureFields":{}
+			}
+		]
+	}]
+}`, http.StatusOK)
+		t.Cleanup(func() {
+			err := resp.Body.Close()
+			require.NoError(t, err)
+		})
+
+		var result apimodels.TestReceiversResult
+		require.NoError(t, json.NewDecoder(resp.Body).Decode(&result))
+
+		require.Len(t, result.Receivers, 1)
+		require.Len(t, result.Receivers[0].Configs, 1)
+		require.Equal(t, apimodels.TestReceiversResult{
+			Receivers: []apimodels.TestReceiverResult{{
+				Name: "receiver-1",
+				Configs: []apimodels.TestReceiverConfigResult{{
+					Name:   "receiver-1",
+					UID:    result.Receivers[0].Configs[0].UID,
+					Status: "ok",
+				}},
+			}},
+			NotifedAt: result.NotifedAt,
+		}, result)
+
+		require.Len(t, mockEmails.emails, 1)
+		require.Equal(t, []string{"example@email.com"}, mockEmails.emails[0].To)
+	})
+
+	t.Run("assert invalid receiver returns 400 Bad Request", func(t *testing.T) {
+		// Setup Grafana and its Database
+		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+			EnableFeatureToggles: []string{"ngalert"},
+		})
+		store := testinfra.SetUpDatabase(t, dir)
+		store.Bus = bus.GetBus()
+		grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
+		createUser(t, store, models.CreateUserCommand{
+			DefaultOrgRole: string(models.ROLE_EDITOR),
+			Login:          "grafana",
+			Password:       "password",
+		})
+
+		oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync")
+		mockEmails := &mockEmailHandler{}
+		bus.AddHandlerCtx("", mockEmails.sendEmailCommandHandlerSync)
+		t.Cleanup(func() {
+			bus.AddHandlerCtx("", oldEmailBus)
+		})
+
+		testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
+		// nolint
+		resp := postRequest(t, testReceiversURL, `{
+	"receivers": [{
+		"name":"receiver-1",
+		"grafana_managed_receiver_configs": [
+			{
+				"uid":"",
+				"name":"receiver-1",
+				"type":"email",
+				"disableResolveMessage":false,
+				"settings":{},
+				"secureFields":{}
+			}
+		]
+	}]
+}`, http.StatusBadRequest)
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			require.NoError(t, resp.Body.Close())
+		})
+
+		var result apimodels.TestReceiversResult
+		require.NoError(t, json.Unmarshal(b, &result))
+		require.Len(t, result.Receivers, 1)
+		require.Len(t, result.Receivers[0].Configs, 1)
+		require.Equal(t, apimodels.TestReceiversResult{
+			Receivers: []apimodels.TestReceiverResult{{
+				Name: "receiver-1",
+				Configs: []apimodels.TestReceiverConfigResult{{
+					Name:   "receiver-1",
+					UID:    result.Receivers[0].Configs[0].UID,
+					Status: "failed",
+					Error:  "the receiver is invalid: failed to validate receiver \"receiver-1\" of type \"email\": could not find addresses in settings",
+				}},
+			}},
+			NotifedAt: result.NotifedAt,
+		}, result)
+	})
+
+	t.Run("assert timed out receiver returns 408 Request Timeout", func(t *testing.T) {
+		// Setup Grafana and its Database
+		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+			EnableFeatureToggles: []string{"ngalert"},
+		})
+		store := testinfra.SetUpDatabase(t, dir)
+		store.Bus = bus.GetBus()
+		grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
+		createUser(t, store, models.CreateUserCommand{
+			DefaultOrgRole: string(models.ROLE_EDITOR),
+			Login:          "grafana",
+			Password:       "password",
+		})
+
+		oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync")
+		mockEmails := &mockEmailHandlerWithTimeout{
+			timeout: 5 * time.Second,
+		}
+		bus.AddHandlerCtx("", mockEmails.sendEmailCommandHandlerSync)
+		t.Cleanup(func() {
+			bus.AddHandlerCtx("", oldEmailBus)
+		})
+
+		testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
+		req, err := http.NewRequest(http.MethodPost, testReceiversURL, strings.NewReader(`{
+	"receivers": [{
+		"name":"receiver-1",
+		"grafana_managed_receiver_configs": [
+			{
+				"uid":"",
+				"name":"receiver-1",
+				"type":"email",
+				"disableResolveMessage":false,
+				"settings":{
+					"addresses":"example@email.com"
+				},
+				"secureFields":{}
+			}
+		]
+	}]
+}`))
+		require.NoError(t, err)
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Request-Timeout", "1")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			require.NoError(t, resp.Body.Close())
+		})
+		require.Equal(t, http.StatusRequestTimeout, resp.StatusCode)
+
+		var result apimodels.TestReceiversResult
+		require.NoError(t, json.NewDecoder(resp.Body).Decode(&result))
+
+		require.Len(t, result.Receivers, 1)
+		require.Len(t, result.Receivers[0].Configs, 1)
+		require.Equal(t, apimodels.TestReceiversResult{
+			Receivers: []apimodels.TestReceiverResult{{
+				Name: "receiver-1",
+				Configs: []apimodels.TestReceiverConfigResult{{
+					Name:   "receiver-1",
+					UID:    result.Receivers[0].Configs[0].UID,
+					Status: "failed",
+					Error:  "the receiver timed out: context deadline exceeded",
+				}},
+			}},
+			NotifedAt: result.NotifedAt,
+		}, result)
+	})
+
+	t.Run("assert multiple different errors returns 207 Multi Status", func(t *testing.T) {
+		// Setup Grafana and its Database
+		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+			EnableFeatureToggles: []string{"ngalert"},
+		})
+		store := testinfra.SetUpDatabase(t, dir)
+		store.Bus = bus.GetBus()
+		grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
+		createUser(t, store, models.CreateUserCommand{
+			DefaultOrgRole: string(models.ROLE_EDITOR),
+			Login:          "grafana",
+			Password:       "password",
+		})
+
+		oldEmailBus := bus.GetHandlerCtx("SendEmailCommandSync")
+		mockEmails := &mockEmailHandlerWithTimeout{
+			timeout: 5 * time.Second,
+		}
+		bus.AddHandlerCtx("", mockEmails.sendEmailCommandHandlerSync)
+		t.Cleanup(func() {
+			bus.AddHandlerCtx("", oldEmailBus)
+		})
+
+		testReceiversURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/receivers/test", grafanaListedAddr)
+		req, err := http.NewRequest(http.MethodPost, testReceiversURL, strings.NewReader(`{
+	"receivers": [{
+		"name":"receiver-1",
+		"grafana_managed_receiver_configs": [
+			{
+				"uid":"",
+				"name":"receiver-1",
+				"type":"email",
+				"disableResolveMessage":false,
+				"settings":{},
+				"secureFields":{}
+			}
+		]
+	}, {
+		"name":"receiver-2",
+		"grafana_managed_receiver_configs": [
+			{
+				"uid":"",
+				"name":"receiver-2",
+				"type":"email",
+				"disableResolveMessage":false,
+				"settings":{
+					"addresses":"example@email.com"
+				},
+				"secureFields":{}
+			}
+		]
+	}]
+}`))
+		require.NoError(t, err)
+		req.Header.Set("Content-Type", "application/json")
+		req.Header.Set("Request-Timeout", "1")
+
+		resp, err := http.DefaultClient.Do(req)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			require.NoError(t, resp.Body.Close())
+		})
+		require.Equal(t, http.StatusMultiStatus, resp.StatusCode)
+
+		var result apimodels.TestReceiversResult
+		require.NoError(t, json.NewDecoder(resp.Body).Decode(&result))
+
+		require.Len(t, result.Receivers, 2)
+		require.Len(t, result.Receivers[0].Configs, 1)
+		require.Len(t, result.Receivers[1].Configs, 1)
+		require.Equal(t, apimodels.TestReceiversResult{
+			Receivers: []apimodels.TestReceiverResult{{
+				Name: "receiver-1",
+				Configs: []apimodels.TestReceiverConfigResult{{
+					Name:   "receiver-1",
+					UID:    result.Receivers[0].Configs[0].UID,
+					Status: "failed",
+					Error:  "the receiver is invalid: failed to validate receiver \"receiver-1\" of type \"email\": could not find addresses in settings",
+				}},
+			}, {
+				Name: "receiver-2",
+				Configs: []apimodels.TestReceiverConfigResult{{
+					Name:   "receiver-2",
+					UID:    result.Receivers[1].Configs[0].UID,
+					Status: "failed",
+					Error:  "the receiver timed out: context deadline exceeded",
+				}},
+			}},
+			NotifedAt: result.NotifedAt,
+		}, result)
+	})
+}
+
 func TestNotificationChannels(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
 		EnableFeatureToggles: []string{"ngalert"},
@@ -391,6 +713,21 @@ func (e *mockEmailHandler) sendEmailCommandHandlerSync(_ context.Context, cmd *m
 	return nil
 }
 
+// mockEmailHandlerWithTimeout blocks until the timeout has expired.
+type mockEmailHandlerWithTimeout struct {
+	mockEmailHandler
+	timeout time.Duration
+}
+
+func (e *mockEmailHandlerWithTimeout) sendEmailCommandHandlerSync(ctx context.Context, cmd *models.SendEmailCommandSync) error {
+	select {
+	case <-time.After(e.timeout):
+		return e.mockEmailHandler.sendEmailCommandHandlerSync(ctx, cmd)
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
 // alertmanagerConfig has the config for all the notification channels
 // that we want to test. It is recommended to use different URL for each
 // channel and have 1 route per channel.