Alerting: Refactor GET/POST alerting config routes to be more extensible (#47229)

* Refactor GET am config to be extensible

* Extract post config route

* Fix tests

* Remove temporary duplication

* Fix broken test due to layer shift

* Fix duplicated error message

* Properly return 400 on config rejection

* Revert weird half method extraction

* Move things to notifier package and avoid redundant interface

* Simplify documentation

* Split encryption service and depend on minimal abstractions

* Properly initialize things all the way up to the composition root

* Encryption -> Crypto

* Address misc feedback

* Missing docstring

* Few more simple polish improvements

* Unify on MultiOrgAlertmanager. Discover bug in existing test

* Fix rebase conflicts

* Misc feedback, renames, docs

* Access crypto hanging off MultiOrgAlertmanager rather than having a separate API to initialize

pkg/services/ngalert/api/api.go

@@ -91,7 +91,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
 	api.RegisterAlertmanagerApiEndpoints(NewForkedAM(
 		api.DatasourceCache,
 		NewLotexAM(proxy, logger),
-		&AlertmanagerSrv{store: api.AlertingStore, mam: api.MultiOrgAlertmanager, secrets: api.SecretsService, log: logger, ac: api.AccessControl},
+		&AlertmanagerSrv{crypto: api.MultiOrgAlertmanager.Crypto, log: logger, ac: api.AccessControl, mam: api.MultiOrgAlertmanager},
 	), m)
 	// Register endpoints for proxying to Prometheus-compatible backends.
 	api.RegisterPrometheusApiEndpoints(NewForkedProm(

pkg/services/ngalert/api/api_alertmanager.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"context"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"net/http"
@@ -15,10 +14,8 @@ import (
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
-	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
-	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/util"
 	"github.com/grafana/grafana/pkg/web"
 )
@@ -29,11 +26,10 @@ const (
 )
 
 type AlertmanagerSrv struct {
-	mam     *notifier.MultiOrgAlertmanager
+	log    log.Logger
-	secrets secrets.Service
+	ac     accesscontrol.AccessControl
-	store   AlertingStore
+	mam    *notifier.MultiOrgAlertmanager
-	log     log.Logger
+	crypto notifier.Crypto
-	ac      accesscontrol.AccessControl
 }
 
 type UnknownReceiverError struct {
@@ -44,81 +40,6 @@ func (e UnknownReceiverError) Error() string {
 	return fmt.Sprintf("unknown receiver: %s", e.UID)
 }
 
-func (srv AlertmanagerSrv) loadSecureSettings(ctx context.Context, orgId int64, receivers []*apimodels.PostableApiReceiver) error {
-	// Get the last known working configuration
-	query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
-	if err := srv.store.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
-		// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
-		if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
-			return fmt.Errorf("failed to get latest configuration: %w", err)
-		}
-	}
-
-	currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
-	if query.Result != nil {
-		currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
-		// If the current config is un-loadable, treat it as if it never existed. Providing a new, valid config should be able to "fix" this state.
-		if err != nil {
-			srv.log.Warn("Last known alertmanager configuration was invalid. Overwriting...")
-		} else {
-			currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
-		}
-	}
-
-	// Copy the previously known secure settings
-	for i, r := range receivers {
-		for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
-			if gr.UID == "" { // new receiver
-				continue
-			}
-
-			cgmr, ok := currentReceiverMap[gr.UID]
-			if !ok {
-				// it tries to update a receiver that didn't previously exist
-				return UnknownReceiverError{UID: gr.UID}
-			}
-
-			// frontend sends only the secure settings that have to be updated
-			// therefore we have to copy from the last configuration only those secure settings not included in the request
-			for key := range cgmr.SecureSettings {
-				_, ok := gr.SecureSettings[key]
-				if !ok {
-					decryptedValue, err := srv.getDecryptedSecret(cgmr, key)
-					if err != nil {
-						return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
-					}
-
-					if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
-						receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
-					}
-
-					receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
-				}
-			}
-		}
-	}
-	return nil
-}
-
-func (srv AlertmanagerSrv) getDecryptedSecret(r *apimodels.PostableGrafanaReceiver, key string) (string, error) {
-	storedValue, ok := r.SecureSettings[key]
-	if !ok {
-		return "", nil
-	}
-
-	decodeValue, err := base64.StdEncoding.DecodeString(storedValue)
-	if err != nil {
-		return "", err
-	}
-
-	decryptedValue, err := srv.secrets.Decrypt(context.Background(), decodeValue)
-	if err != nil {
-		return "", err
-	}
-
-	return string(decryptedValue), nil
-}
-
 func (srv AlertmanagerSrv) RouteGetAMStatus(c *models.ReqContext) response.Response {
 	am, errResp := srv.AlertmanagerFor(c.OrgId)
 	if errResp != nil {
@@ -192,59 +113,14 @@ func (srv AlertmanagerSrv) RouteDeleteSilence(c *models.ReqContext) response.Res
 }
 
 func (srv AlertmanagerSrv) RouteGetAlertingConfig(c *models.ReqContext) response.Response {
-	query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: c.OrgId}
+	config, err := srv.mam.GetAlertmanagerConfiguration(c.Req.Context(), c.OrgId)
-	if err := srv.store.GetLatestAlertmanagerConfiguration(c.Req.Context(), &query); err != nil {
+	if err != nil {
 		if errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
 			return ErrResp(http.StatusNotFound, err, "")
 		}
-		return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration")
+		return ErrResp(http.StatusInternalServerError, err, err.Error())
 	}
-
+	return response.JSON(http.StatusOK, config)
-	cfg, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
-	if err != nil {
-		return ErrResp(http.StatusInternalServerError, err, "failed to unmarshal alertmanager configuration")
-	}
-
-	result := apimodels.GettableUserConfig{
-		TemplateFiles: cfg.TemplateFiles,
-		AlertmanagerConfig: apimodels.GettableApiAlertingConfig{
-			Config: cfg.AlertmanagerConfig.Config,
-		},
-	}
-	for _, recv := range cfg.AlertmanagerConfig.Receivers {
-		receivers := make([]*apimodels.GettableGrafanaReceiver, 0, len(recv.PostableGrafanaReceivers.GrafanaManagedReceivers))
-		for _, pr := range recv.PostableGrafanaReceivers.GrafanaManagedReceivers {
-			secureFields := make(map[string]bool, len(pr.SecureSettings))
-			for k := range pr.SecureSettings {
-				decryptedValue, err := srv.getDecryptedSecret(pr, k)
-				if err != nil {
-					return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", k)
-				}
-				if decryptedValue == "" {
-					continue
-				}
-				secureFields[k] = true
-			}
-			gr := apimodels.GettableGrafanaReceiver{
-				UID:                   pr.UID,
-				Name:                  pr.Name,
-				Type:                  pr.Type,
-				DisableResolveMessage: pr.DisableResolveMessage,
-				Settings:              pr.Settings,
-				SecureFields:          secureFields,
-			}
-			receivers = append(receivers, &gr)
-		}
-		gettableApiReceiver := apimodels.GettableApiReceiver{
-			GettableGrafanaReceivers: apimodels.GettableGrafanaReceivers{
-				GrafanaManagedReceivers: receivers,
-			},
-		}
-		gettableApiReceiver.Name = recv.Name
-		result.AlertmanagerConfig.Receivers = append(result.AlertmanagerConfig.Receivers, &gettableApiReceiver)
-	}
-
-	return response.JSON(http.StatusOK, result)
 }
 
 func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *models.ReqContext) response.Response {
@@ -334,41 +210,26 @@ func (srv AlertmanagerSrv) RouteGetSilences(c *models.ReqContext) response.Respo
 }
 
 func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body apimodels.PostableUserConfig) response.Response {
-	// Get the last known working configuration
+	err := srv.mam.ApplyAlertmanagerConfiguration(c.Req.Context(), c.OrgId, body)
-	query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: c.OrgId}
+	if err == nil {
-	if err := srv.store.GetLatestAlertmanagerConfiguration(c.Req.Context(), &query); err != nil {
+		return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"})
-		// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
+	}
-		if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
+	var unknownReceiverError notifier.UnknownReceiverError
-			return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration")
+	if errors.As(err, &unknownReceiverError) {
-		}
+		return ErrResp(http.StatusBadRequest, unknownReceiverError, "")
+	}
+	var configRejectedError notifier.AlertmanagerConfigRejectedError
+	if errors.As(err, &configRejectedError) {
+		return ErrResp(http.StatusBadRequest, configRejectedError, "")
+	}
+	if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
+		return response.Error(http.StatusNotFound, err.Error(), err)
+	}
+	if errors.Is(err, notifier.ErrAlertmanagerNotReady) {
+		return response.Error(http.StatusConflict, err.Error(), err)
 	}
 
-	if err := srv.loadSecureSettings(c.Req.Context(), c.OrgId, body.AlertmanagerConfig.Receivers); err != nil {
+	return ErrResp(http.StatusInternalServerError, err, "")
-		var unknownReceiverError UnknownReceiverError
-		if errors.As(err, &unknownReceiverError) {
-			return ErrResp(http.StatusBadRequest, err, "")
-		}
-		return ErrResp(http.StatusInternalServerError, err, "")
-	}
-
-	if err := body.ProcessConfig(srv.secrets.Encrypt); err != nil {
-		return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
-	}
-
-	am, errResp := srv.AlertmanagerFor(c.OrgId)
-	if errResp != nil {
-		// It's okay if the alertmanager isn't ready yet, we're changing its config anyway.
-		if !errors.Is(errResp.Err(), notifier.ErrAlertmanagerNotReady) {
-			return errResp
-		}
-	}
-
-	if err := am.SaveAndApplyConfig(c.Req.Context(), &body); err != nil {
-		srv.log.Error("unable to save and apply alertmanager configuration", "err", err)
-		return ErrResp(http.StatusBadRequest, err, "failed to save and apply Alertmanager configuration")
-	}
-
-	return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"})
 }
 
 func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.PostableAlerts) response.Response {
@@ -376,7 +237,7 @@ func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.P
 }
 
 func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body apimodels.TestReceiversConfigBodyParams) response.Response {
-	if err := srv.loadSecureSettings(c.Req.Context(), c.OrgId, body.Receivers); err != nil {
+	if err := srv.crypto.LoadSecureSettings(c.Req.Context(), c.OrgId, body.Receivers); err != nil {
 		var unknownReceiverError UnknownReceiverError
 		if errors.As(err, &unknownReceiverError) {
 			return ErrResp(http.StatusBadRequest, err, "")
@@ -384,7 +245,7 @@ func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body api
 		return ErrResp(http.StatusInternalServerError, err, "")
 	}
 
-	if err := body.ProcessConfig(srv.secrets.Encrypt); err != nil {
+	if err := body.ProcessConfig(srv.crypto.Encrypt); err != nil {
 		return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
 	}
 
@@ -514,7 +375,6 @@ func (srv AlertmanagerSrv) AlertmanagerFor(orgID int64) (Alertmanager, *response
 	if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
 		return nil, response.Error(http.StatusNotFound, err.Error(), err)
 	}
-
 	if errors.Is(err, notifier.ErrAlertmanagerNotReady) {
 		return am, response.Error(http.StatusConflict, err.Error(), err)
 	}

pkg/services/ngalert/api/api_alertmanager_test.go

@@ -357,7 +357,8 @@ func createSut(t *testing.T, accessControl accesscontrol.AccessControl) Alertman
 		accessControl = acMock.New().WithDisabled()
 	}
 	log := log.NewNopLogger()
-	return AlertmanagerSrv{mam: mam, store: &configStore, secrets: secrets, ac: accessControl, log: log}
+	crypto := notifier.NewCrypto(secrets, &configStore, log)
+	return AlertmanagerSrv{mam: mam, crypto: crypto, ac: accessControl, log: log}
 }
 
 func createAmConfigRequest(t *testing.T) apimodels.PostableUserConfig {
@@ -395,7 +396,7 @@ func createMultiOrgAlertmanager(t *testing.T) *notifier.MultiOrgAlertmanager {
 		}, // do not poll in tests.
 	}
 
-	mam, err := notifier.NewMultiOrgAlertmanager(cfg, &configStore, &orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
+	mam, err := notifier.NewMultiOrgAlertmanager(cfg, &configStore, &orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
 	require.NoError(t, err)
 	err = mam.LoadAndSyncAlertmanagersForOrgs(context.Background())
 	require.NoError(t, err)

pkg/services/ngalert/api/tooling/definitions/alertmanager.go

@@ -18,6 +18,7 @@ import (
 	"gopkg.in/yaml.v3"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
+	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/secrets"
 	"github.com/grafana/grafana/pkg/util"
 )
@@ -712,6 +713,8 @@ type Route struct {
 	GroupWait      *model.Duration `yaml:"group_wait,omitempty" json:"group_wait,omitempty"`
 	GroupInterval  *model.Duration `yaml:"group_interval,omitempty" json:"group_interval,omitempty"`
 	RepeatInterval *model.Duration `yaml:"repeat_interval,omitempty" json:"repeat_interval,omitempty"`
+
+	Provenance models.Provenance `yaml:"provenance,omitempty" json:"provenance,omitempty"`
 }
 
 // UnmarshalYAML implements the yaml.Unmarshaler interface for Route. This is a copy of alertmanager's upstream except it removes validation on the label key.

pkg/services/ngalert/ngalert.go

@@ -99,7 +99,7 @@ func (ng *AlertNG) init() error {
 
 	decryptFn := ng.SecretsService.GetDecryptedValue
 	multiOrgMetrics := ng.Metrics.GetMultiOrgAlertmanagerMetrics()
-	ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, decryptFn, multiOrgMetrics, ng.NotificationService, log.New("ngalert.multiorg.alertmanager"))
+	ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, decryptFn, multiOrgMetrics, ng.NotificationService, log.New("ngalert.multiorg.alertmanager"), ng.SecretsService)
 	if err != nil {
 		return err
 	}

pkg/services/ngalert/notifier/alertmanager_config.go

@@ -0,0 +1,119 @@
+package notifier
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/services/ngalert/store"
+)
+
+type UnknownReceiverError struct {
+	UID string
+}
+
+func (e UnknownReceiverError) Error() string {
+	return fmt.Sprintf("unknown receiver: %s", e.UID)
+}
+
+type AlertmanagerConfigRejectedError struct {
+	Inner error
+}
+
+func (e AlertmanagerConfigRejectedError) Error() string {
+	return fmt.Sprintf("failed to save and apply Alertmanager configuration: %s", e.Inner.Error())
+}
+
+type configurationStore interface {
+	GetLatestAlertmanagerConfiguration(ctx context.Context, query *models.GetLatestAlertmanagerConfigurationQuery) error
+}
+
+func (moa *MultiOrgAlertmanager) GetAlertmanagerConfiguration(ctx context.Context, org int64) (definitions.GettableUserConfig, error) {
+	query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: org}
+	err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, &query)
+	if err != nil {
+		return definitions.GettableUserConfig{}, fmt.Errorf("failed to get latest configuration: %w", err)
+	}
+	cfg, err := Load([]byte(query.Result.AlertmanagerConfiguration))
+	if err != nil {
+		return definitions.GettableUserConfig{}, fmt.Errorf("failed to unmarshal alertmanager configuration: %w", err)
+	}
+
+	result := definitions.GettableUserConfig{
+		TemplateFiles: cfg.TemplateFiles,
+		AlertmanagerConfig: definitions.GettableApiAlertingConfig{
+			Config: cfg.AlertmanagerConfig.Config,
+		},
+	}
+
+	for _, recv := range cfg.AlertmanagerConfig.Receivers {
+		receivers := make([]*definitions.GettableGrafanaReceiver, 0, len(recv.PostableGrafanaReceivers.GrafanaManagedReceivers))
+		for _, pr := range recv.PostableGrafanaReceivers.GrafanaManagedReceivers {
+			secureFields := make(map[string]bool, len(pr.SecureSettings))
+			for k := range pr.SecureSettings {
+				decryptedValue, err := moa.Crypto.getDecryptedSecret(pr, k)
+				if err != nil {
+					return definitions.GettableUserConfig{}, fmt.Errorf("failed to decrypt stored secure setting: %w", err)
+				}
+				if decryptedValue == "" {
+					continue
+				}
+				secureFields[k] = true
+			}
+			gr := definitions.GettableGrafanaReceiver{
+				UID:                   pr.UID,
+				Name:                  pr.Name,
+				Type:                  pr.Type,
+				DisableResolveMessage: pr.DisableResolveMessage,
+				Settings:              pr.Settings,
+				SecureFields:          secureFields,
+			}
+			receivers = append(receivers, &gr)
+		}
+		gettableApiReceiver := definitions.GettableApiReceiver{
+			GettableGrafanaReceivers: definitions.GettableGrafanaReceivers{
+				GrafanaManagedReceivers: receivers,
+			},
+		}
+		gettableApiReceiver.Name = recv.Name
+		result.AlertmanagerConfig.Receivers = append(result.AlertmanagerConfig.Receivers, &gettableApiReceiver)
+	}
+
+	return result, nil
+}
+
+func (moa *MultiOrgAlertmanager) ApplyAlertmanagerConfiguration(ctx context.Context, org int64, config definitions.PostableUserConfig) error {
+	// Get the last known working configuration
+	query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: org}
+	if err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
+		// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
+		if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
+			return fmt.Errorf("failed to get latest configuration %w", err)
+		}
+	}
+
+	if err := moa.Crypto.LoadSecureSettings(ctx, org, config.AlertmanagerConfig.Receivers); err != nil {
+		return err
+	}
+
+	if err := config.ProcessConfig(moa.Crypto.Encrypt); err != nil {
+		return fmt.Errorf("failed to post process Alertmanager configuration: %w", err)
+	}
+
+	am, err := moa.AlertmanagerFor(org)
+	if err != nil {
+		// It's okay if the alertmanager isn't ready yet, we're changing its config anyway.
+		if !errors.Is(err, ErrAlertmanagerNotReady) {
+			return err
+		}
+	}
+
+	if err := am.SaveAndApplyConfig(ctx, &config); err != nil {
+		moa.logger.Error("unable to save and apply alertmanager configuration", "err", err)
+		return AlertmanagerConfigRejectedError{err}
+	}
+
+	return nil
+}

pkg/services/ngalert/notifier/crypto.go

@@ -0,0 +1,118 @@
+package notifier
+
+import (
+	"context"
+	"encoding/base64"
+	"errors"
+	"fmt"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/services/ngalert/store"
+	"github.com/grafana/grafana/pkg/services/secrets"
+)
+
+// Crypto allows decryption of Alertmanager Configuration and encryption of arbitrary payloads.
+type Crypto interface {
+	LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error
+	Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error)
+
+	getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error)
+}
+
+// alertmanagerCrypto implements decryption of Alertmanager configuration and encryption of arbitrary payloads based on Grafana's encryptions.
+type alertmanagerCrypto struct {
+	secrets secrets.Service
+	configs configurationStore
+	log     log.Logger
+}
+
+func NewCrypto(secrets secrets.Service, configs configurationStore, log log.Logger) Crypto {
+	return &alertmanagerCrypto{
+		secrets: secrets,
+		configs: configs,
+		log:     log,
+	}
+}
+
+// LoadSecureSettings adds the corresponding unencrypted secrets stored to the list of input receivers.
+func (c *alertmanagerCrypto) LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error {
+	// Get the last known working configuration.
+	query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
+	if err := c.configs.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
+		// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one.
+		if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
+			return fmt.Errorf("failed to get latest configuration: %w", err)
+		}
+	}
+
+	currentReceiverMap := make(map[string]*definitions.PostableGrafanaReceiver)
+	if query.Result != nil {
+		currentConfig, err := Load([]byte(query.Result.AlertmanagerConfiguration))
+		// If the current config is un-loadable, treat it as if it never existed. Providing a new, valid config should be able to "fix" this state.
+		if err != nil {
+			c.log.Warn("Last known alertmanager configuration was invalid. Overwriting...")
+		} else {
+			currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
+		}
+	}
+
+	// Copy the previously known secure settings.
+	for i, r := range receivers {
+		for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
+			if gr.UID == "" { // new receiver
+				continue
+			}
+
+			cgmr, ok := currentReceiverMap[gr.UID]
+			if !ok {
+				// It tries to update a receiver that didn't previously exist
+				return UnknownReceiverError{UID: gr.UID}
+			}
+
+			// Frontend sends only the secure settings that have to be updated
+			// Therefore we have to copy from the last configuration only those secure settings not included in the request
+			for key := range cgmr.SecureSettings {
+				_, ok := gr.SecureSettings[key]
+				if !ok {
+					decryptedValue, err := c.getDecryptedSecret(cgmr, key)
+					if err != nil {
+						return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
+					}
+
+					if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
+						receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
+					}
+
+					receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func (c *alertmanagerCrypto) getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error) {
+	storedValue, ok := r.SecureSettings[key]
+	if !ok {
+		return "", nil
+	}
+
+	decodeValue, err := base64.StdEncoding.DecodeString(storedValue)
+	if err != nil {
+		return "", err
+	}
+
+	decryptedValue, err := c.secrets.Decrypt(context.Background(), decodeValue)
+	if err != nil {
+		return "", err
+	}
+
+	return string(decryptedValue), nil
+}
+
+// Encrypt delegates encryption to secrets.Service.
+func (c *alertmanagerCrypto) Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) {
+	return c.secrets.Encrypt(ctx, payload, opt)
+}

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier/channels"
 	"github.com/grafana/grafana/pkg/services/notifications"
+	"github.com/grafana/grafana/pkg/services/secrets"
 
 	"github.com/prometheus/alertmanager/cluster"
 	"github.com/prometheus/client_golang/prometheus"
@@ -29,6 +30,8 @@ var (
 )
 
 type MultiOrgAlertmanager struct {
+	Crypto Crypto
+
 	alertmanagersMtx sync.RWMutex
 	alertmanagers    map[int64]*Alertmanager
 
@@ -51,9 +54,11 @@ type MultiOrgAlertmanager struct {
 
 func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore,
 	kvStore kvstore.KVStore, decryptFn channels.GetDecryptedValueFn, m *metrics.MultiOrgAlertmanager,
-	ns notifications.Service, l log.Logger,
+	ns notifications.Service, l log.Logger, s secrets.Service,
 ) (*MultiOrgAlertmanager, error) {
 	moa := &MultiOrgAlertmanager{
+		Crypto: NewCrypto(s, configStore, l),
+
 		logger:        l,
 		settings:      cfg,
 		alertmanagers: map[int64]*Alertmanager{},

pkg/services/ngalert/notifier/multiorg_alertmanager_test.go

@@ -44,7 +44,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
 			DisabledOrgs:                   map[int64]struct{}{5: {}},
 		}, // do not poll in tests.
 	}
-	mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
+	mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
 	require.NoError(t, err)
 	ctx := context.Background()
 
@@ -168,7 +168,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T)
 			DefaultConfiguration:           setting.GetAlertmanagerDefaultConfiguration(),
 		}, // do not poll in tests.
 	}
-	mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
+	mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
 	require.NoError(t, err)
 	ctx := context.Background()
 
@@ -218,7 +218,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
 	decryptFn := secretsService.GetDecryptedValue
 	reg := prometheus.NewPedanticRegistry()
 	m := metrics.NewNGAlert(reg)
-	mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
+	mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
 	require.NoError(t, err)
 	ctx := context.Background()
 

pkg/services/ngalert/schedule/schedule_unit_test.go

@@ -1015,7 +1015,7 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac
 	m := metrics.NewNGAlert(registry)
 	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
 	decryptFn := secretsService.GetDecryptedValue
-	moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, &notifier.FakeConfigStore{}, &notifier.FakeOrgStore{}, &notifier.FakeKVStore{}, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
+	moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, &notifier.FakeConfigStore{}, &notifier.FakeOrgStore{}, &notifier.FakeKVStore{}, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
 	require.NoError(t, err)
 
 	schedCfg := SchedulerCfg{