Alerting: Decrypt secrets before sending configuration to the remote Alertmanager (#83640)

* (WIP) Alerting: Decrypt secrets before sending configuration to the remote Alertmanager

* refactor, fix tests

* test decrypting secrets

* tidy up

* test SendConfiguration, quote keys, refactor tests

* make linter happy

* decrypt configuration before comparing

* copy configuration struct before decrypting

* reduce diff in TestCompareAndSendConfiguration

* clean up remote/alertmanager.go

* make linter happy

* avoid serializing into JSON to copy struct

* codeowners

pkg/services/ngalert/api/tooling/definitions/alertmanager.go

@@ -2,12 +2,14 @@ package definitions
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"time"
 
 	"github.com/go-openapi/strfmt"
 	"github.com/grafana/alerting/definition"
+	"github.com/mohae/deepcopy"
 	amv2 "github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/common/model"
@@ -615,6 +617,34 @@ func (c *PostableUserConfig) validate() error {
 	return nil
 }
 
+// Decrypt returns a copy of the configuration struct with decrypted secure settings in receivers.
+func (c *PostableUserConfig) Decrypt(decryptFn func(payload []byte) ([]byte, error)) (PostableUserConfig, error) {
+	newCfg, ok := deepcopy.Copy(c).(*PostableUserConfig)
+	if !ok {
+		return PostableUserConfig{}, fmt.Errorf("failed to copy config")
+	}
+
+	// Iterate through receivers and decrypt secure settings.
+	for _, rcv := range newCfg.AlertmanagerConfig.Receivers {
+		for _, gmr := range rcv.PostableGrafanaReceivers.GrafanaManagedReceivers {
+			for k, v := range gmr.SecureSettings {
+				decoded, err := base64.StdEncoding.DecodeString(v)
+				if err != nil {
+					return PostableUserConfig{}, fmt.Errorf("failed to decode value for key '%s': %w", k, err)
+				}
+
+				decrypted, err := decryptFn(decoded)
+				if err != nil {
+					return PostableUserConfig{}, fmt.Errorf("failed to decrypt value for key '%s': %w", k, err)
+				}
+
+				gmr.SecureSettings[k] = string(decrypted)
+			}
+		}
+	}
+	return *newCfg, nil
+}
+
 // GetGrafanaReceiverMap returns a map that associates UUIDs to grafana receivers
 func (c *PostableUserConfig) GetGrafanaReceiverMap() map[string]*PostableGrafanaReceiver {
 	UIDs := make(map[string]*PostableGrafanaReceiver)