mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Refactor GET/POST alerting config routes to be more extensible (#47229)
* Refactor GET am config to be extensible * Extract post config route * Fix tests * Remove temporary duplication * Fix broken test due to layer shift * Fix duplicated error message * Properly return 400 on config rejection * Revert weird half method extraction * Move things to notifier package and avoid redundant interface * Simplify documentation * Split encryption service and depend on minimal abstractions * Properly initialize things all the way up to the composition root * Encryption -> Crypto * Address misc feedback * Missing docstring * Few more simple polish improvements * Unify on MultiOrgAlertmanager. Discover bug in existing test * Fix rebase conflicts * Misc feedback, renames, docs * Access crypto hanging off MultiOrgAlertmanager rather than having a separate API to initialize
This commit is contained in:
parent
a126c07e54
commit
758364e78b
@ -91,7 +91,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
|
||||
api.RegisterAlertmanagerApiEndpoints(NewForkedAM(
|
||||
api.DatasourceCache,
|
||||
NewLotexAM(proxy, logger),
|
||||
&AlertmanagerSrv{store: api.AlertingStore, mam: api.MultiOrgAlertmanager, secrets: api.SecretsService, log: logger, ac: api.AccessControl},
|
||||
&AlertmanagerSrv{crypto: api.MultiOrgAlertmanager.Crypto, log: logger, ac: api.AccessControl, mam: api.MultiOrgAlertmanager},
|
||||
), m)
|
||||
// Register endpoints for proxying to Prometheus-compatible backends.
|
||||
api.RegisterPrometheusApiEndpoints(NewForkedProm(
|
||||
|
@ -2,7 +2,6 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@ -15,10 +14,8 @@ import (
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/accesscontrol"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/store"
|
||||
"github.com/grafana/grafana/pkg/services/secrets"
|
||||
"github.com/grafana/grafana/pkg/util"
|
||||
"github.com/grafana/grafana/pkg/web"
|
||||
)
|
||||
@ -29,11 +26,10 @@ const (
|
||||
)
|
||||
|
||||
type AlertmanagerSrv struct {
|
||||
mam *notifier.MultiOrgAlertmanager
|
||||
secrets secrets.Service
|
||||
store AlertingStore
|
||||
log log.Logger
|
||||
ac accesscontrol.AccessControl
|
||||
log log.Logger
|
||||
ac accesscontrol.AccessControl
|
||||
mam *notifier.MultiOrgAlertmanager
|
||||
crypto notifier.Crypto
|
||||
}
|
||||
|
||||
type UnknownReceiverError struct {
|
||||
@ -44,81 +40,6 @@ func (e UnknownReceiverError) Error() string {
|
||||
return fmt.Sprintf("unknown receiver: %s", e.UID)
|
||||
}
|
||||
|
||||
func (srv AlertmanagerSrv) loadSecureSettings(ctx context.Context, orgId int64, receivers []*apimodels.PostableApiReceiver) error {
|
||||
// Get the last known working configuration
|
||||
query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
|
||||
if err := srv.store.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
|
||||
// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
|
||||
if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
|
||||
return fmt.Errorf("failed to get latest configuration: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
currentReceiverMap := make(map[string]*apimodels.PostableGrafanaReceiver)
|
||||
if query.Result != nil {
|
||||
currentConfig, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
|
||||
// If the current config is un-loadable, treat it as if it never existed. Providing a new, valid config should be able to "fix" this state.
|
||||
if err != nil {
|
||||
srv.log.Warn("Last known alertmanager configuration was invalid. Overwriting...")
|
||||
} else {
|
||||
currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
|
||||
}
|
||||
}
|
||||
|
||||
// Copy the previously known secure settings
|
||||
for i, r := range receivers {
|
||||
for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
|
||||
if gr.UID == "" { // new receiver
|
||||
continue
|
||||
}
|
||||
|
||||
cgmr, ok := currentReceiverMap[gr.UID]
|
||||
if !ok {
|
||||
// it tries to update a receiver that didn't previously exist
|
||||
return UnknownReceiverError{UID: gr.UID}
|
||||
}
|
||||
|
||||
// frontend sends only the secure settings that have to be updated
|
||||
// therefore we have to copy from the last configuration only those secure settings not included in the request
|
||||
for key := range cgmr.SecureSettings {
|
||||
_, ok := gr.SecureSettings[key]
|
||||
if !ok {
|
||||
decryptedValue, err := srv.getDecryptedSecret(cgmr, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
|
||||
}
|
||||
|
||||
if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
|
||||
receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
|
||||
}
|
||||
|
||||
receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (srv AlertmanagerSrv) getDecryptedSecret(r *apimodels.PostableGrafanaReceiver, key string) (string, error) {
|
||||
storedValue, ok := r.SecureSettings[key]
|
||||
if !ok {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
decodeValue, err := base64.StdEncoding.DecodeString(storedValue)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
decryptedValue, err := srv.secrets.Decrypt(context.Background(), decodeValue)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(decryptedValue), nil
|
||||
}
|
||||
|
||||
func (srv AlertmanagerSrv) RouteGetAMStatus(c *models.ReqContext) response.Response {
|
||||
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
||||
if errResp != nil {
|
||||
@ -192,59 +113,14 @@ func (srv AlertmanagerSrv) RouteDeleteSilence(c *models.ReqContext) response.Res
|
||||
}
|
||||
|
||||
func (srv AlertmanagerSrv) RouteGetAlertingConfig(c *models.ReqContext) response.Response {
|
||||
query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: c.OrgId}
|
||||
if err := srv.store.GetLatestAlertmanagerConfiguration(c.Req.Context(), &query); err != nil {
|
||||
config, err := srv.mam.GetAlertmanagerConfiguration(c.Req.Context(), c.OrgId)
|
||||
if err != nil {
|
||||
if errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
|
||||
return ErrResp(http.StatusNotFound, err, "")
|
||||
}
|
||||
return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration")
|
||||
return ErrResp(http.StatusInternalServerError, err, err.Error())
|
||||
}
|
||||
|
||||
cfg, err := notifier.Load([]byte(query.Result.AlertmanagerConfiguration))
|
||||
if err != nil {
|
||||
return ErrResp(http.StatusInternalServerError, err, "failed to unmarshal alertmanager configuration")
|
||||
}
|
||||
|
||||
result := apimodels.GettableUserConfig{
|
||||
TemplateFiles: cfg.TemplateFiles,
|
||||
AlertmanagerConfig: apimodels.GettableApiAlertingConfig{
|
||||
Config: cfg.AlertmanagerConfig.Config,
|
||||
},
|
||||
}
|
||||
for _, recv := range cfg.AlertmanagerConfig.Receivers {
|
||||
receivers := make([]*apimodels.GettableGrafanaReceiver, 0, len(recv.PostableGrafanaReceivers.GrafanaManagedReceivers))
|
||||
for _, pr := range recv.PostableGrafanaReceivers.GrafanaManagedReceivers {
|
||||
secureFields := make(map[string]bool, len(pr.SecureSettings))
|
||||
for k := range pr.SecureSettings {
|
||||
decryptedValue, err := srv.getDecryptedSecret(pr, k)
|
||||
if err != nil {
|
||||
return ErrResp(http.StatusInternalServerError, err, "failed to decrypt stored secure setting: %s", k)
|
||||
}
|
||||
if decryptedValue == "" {
|
||||
continue
|
||||
}
|
||||
secureFields[k] = true
|
||||
}
|
||||
gr := apimodels.GettableGrafanaReceiver{
|
||||
UID: pr.UID,
|
||||
Name: pr.Name,
|
||||
Type: pr.Type,
|
||||
DisableResolveMessage: pr.DisableResolveMessage,
|
||||
Settings: pr.Settings,
|
||||
SecureFields: secureFields,
|
||||
}
|
||||
receivers = append(receivers, &gr)
|
||||
}
|
||||
gettableApiReceiver := apimodels.GettableApiReceiver{
|
||||
GettableGrafanaReceivers: apimodels.GettableGrafanaReceivers{
|
||||
GrafanaManagedReceivers: receivers,
|
||||
},
|
||||
}
|
||||
gettableApiReceiver.Name = recv.Name
|
||||
result.AlertmanagerConfig.Receivers = append(result.AlertmanagerConfig.Receivers, &gettableApiReceiver)
|
||||
}
|
||||
|
||||
return response.JSON(http.StatusOK, result)
|
||||
return response.JSON(http.StatusOK, config)
|
||||
}
|
||||
|
||||
func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *models.ReqContext) response.Response {
|
||||
@ -334,41 +210,26 @@ func (srv AlertmanagerSrv) RouteGetSilences(c *models.ReqContext) response.Respo
|
||||
}
|
||||
|
||||
func (srv AlertmanagerSrv) RoutePostAlertingConfig(c *models.ReqContext, body apimodels.PostableUserConfig) response.Response {
|
||||
// Get the last known working configuration
|
||||
query := ngmodels.GetLatestAlertmanagerConfigurationQuery{OrgID: c.OrgId}
|
||||
if err := srv.store.GetLatestAlertmanagerConfiguration(c.Req.Context(), &query); err != nil {
|
||||
// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
|
||||
if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
|
||||
return ErrResp(http.StatusInternalServerError, err, "failed to get latest configuration")
|
||||
}
|
||||
err := srv.mam.ApplyAlertmanagerConfiguration(c.Req.Context(), c.OrgId, body)
|
||||
if err == nil {
|
||||
return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"})
|
||||
}
|
||||
var unknownReceiverError notifier.UnknownReceiverError
|
||||
if errors.As(err, &unknownReceiverError) {
|
||||
return ErrResp(http.StatusBadRequest, unknownReceiverError, "")
|
||||
}
|
||||
var configRejectedError notifier.AlertmanagerConfigRejectedError
|
||||
if errors.As(err, &configRejectedError) {
|
||||
return ErrResp(http.StatusBadRequest, configRejectedError, "")
|
||||
}
|
||||
if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
|
||||
return response.Error(http.StatusNotFound, err.Error(), err)
|
||||
}
|
||||
if errors.Is(err, notifier.ErrAlertmanagerNotReady) {
|
||||
return response.Error(http.StatusConflict, err.Error(), err)
|
||||
}
|
||||
|
||||
if err := srv.loadSecureSettings(c.Req.Context(), c.OrgId, body.AlertmanagerConfig.Receivers); err != nil {
|
||||
var unknownReceiverError UnknownReceiverError
|
||||
if errors.As(err, &unknownReceiverError) {
|
||||
return ErrResp(http.StatusBadRequest, err, "")
|
||||
}
|
||||
return ErrResp(http.StatusInternalServerError, err, "")
|
||||
}
|
||||
|
||||
if err := body.ProcessConfig(srv.secrets.Encrypt); err != nil {
|
||||
return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
|
||||
}
|
||||
|
||||
am, errResp := srv.AlertmanagerFor(c.OrgId)
|
||||
if errResp != nil {
|
||||
// It's okay if the alertmanager isn't ready yet, we're changing its config anyway.
|
||||
if !errors.Is(errResp.Err(), notifier.ErrAlertmanagerNotReady) {
|
||||
return errResp
|
||||
}
|
||||
}
|
||||
|
||||
if err := am.SaveAndApplyConfig(c.Req.Context(), &body); err != nil {
|
||||
srv.log.Error("unable to save and apply alertmanager configuration", "err", err)
|
||||
return ErrResp(http.StatusBadRequest, err, "failed to save and apply Alertmanager configuration")
|
||||
}
|
||||
|
||||
return response.JSON(http.StatusAccepted, util.DynMap{"message": "configuration created"})
|
||||
return ErrResp(http.StatusInternalServerError, err, "")
|
||||
}
|
||||
|
||||
func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.PostableAlerts) response.Response {
|
||||
@ -376,7 +237,7 @@ func (srv AlertmanagerSrv) RoutePostAMAlerts(_ *models.ReqContext, _ apimodels.P
|
||||
}
|
||||
|
||||
func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body apimodels.TestReceiversConfigBodyParams) response.Response {
|
||||
if err := srv.loadSecureSettings(c.Req.Context(), c.OrgId, body.Receivers); err != nil {
|
||||
if err := srv.crypto.LoadSecureSettings(c.Req.Context(), c.OrgId, body.Receivers); err != nil {
|
||||
var unknownReceiverError UnknownReceiverError
|
||||
if errors.As(err, &unknownReceiverError) {
|
||||
return ErrResp(http.StatusBadRequest, err, "")
|
||||
@ -384,7 +245,7 @@ func (srv AlertmanagerSrv) RoutePostTestReceivers(c *models.ReqContext, body api
|
||||
return ErrResp(http.StatusInternalServerError, err, "")
|
||||
}
|
||||
|
||||
if err := body.ProcessConfig(srv.secrets.Encrypt); err != nil {
|
||||
if err := body.ProcessConfig(srv.crypto.Encrypt); err != nil {
|
||||
return ErrResp(http.StatusInternalServerError, err, "failed to post process Alertmanager configuration")
|
||||
}
|
||||
|
||||
@ -514,7 +375,6 @@ func (srv AlertmanagerSrv) AlertmanagerFor(orgID int64) (Alertmanager, *response
|
||||
if errors.Is(err, notifier.ErrNoAlertmanagerForOrg) {
|
||||
return nil, response.Error(http.StatusNotFound, err.Error(), err)
|
||||
}
|
||||
|
||||
if errors.Is(err, notifier.ErrAlertmanagerNotReady) {
|
||||
return am, response.Error(http.StatusConflict, err.Error(), err)
|
||||
}
|
||||
|
@ -357,7 +357,8 @@ func createSut(t *testing.T, accessControl accesscontrol.AccessControl) Alertman
|
||||
accessControl = acMock.New().WithDisabled()
|
||||
}
|
||||
log := log.NewNopLogger()
|
||||
return AlertmanagerSrv{mam: mam, store: &configStore, secrets: secrets, ac: accessControl, log: log}
|
||||
crypto := notifier.NewCrypto(secrets, &configStore, log)
|
||||
return AlertmanagerSrv{mam: mam, crypto: crypto, ac: accessControl, log: log}
|
||||
}
|
||||
|
||||
func createAmConfigRequest(t *testing.T) apimodels.PostableUserConfig {
|
||||
@ -395,7 +396,7 @@ func createMultiOrgAlertmanager(t *testing.T) *notifier.MultiOrgAlertmanager {
|
||||
}, // do not poll in tests.
|
||||
}
|
||||
|
||||
mam, err := notifier.NewMultiOrgAlertmanager(cfg, &configStore, &orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
|
||||
mam, err := notifier.NewMultiOrgAlertmanager(cfg, &configStore, &orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
|
||||
require.NoError(t, err)
|
||||
err = mam.LoadAndSyncAlertmanagersForOrgs(context.Background())
|
||||
require.NoError(t, err)
|
||||
|
@ -18,6 +18,7 @@ import (
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
"github.com/grafana/grafana/pkg/services/secrets"
|
||||
"github.com/grafana/grafana/pkg/util"
|
||||
)
|
||||
@ -712,6 +713,8 @@ type Route struct {
|
||||
GroupWait *model.Duration `yaml:"group_wait,omitempty" json:"group_wait,omitempty"`
|
||||
GroupInterval *model.Duration `yaml:"group_interval,omitempty" json:"group_interval,omitempty"`
|
||||
RepeatInterval *model.Duration `yaml:"repeat_interval,omitempty" json:"repeat_interval,omitempty"`
|
||||
|
||||
Provenance models.Provenance `yaml:"provenance,omitempty" json:"provenance,omitempty"`
|
||||
}
|
||||
|
||||
// UnmarshalYAML implements the yaml.Unmarshaler interface for Route. This is a copy of alertmanager's upstream except it removes validation on the label key.
|
||||
|
@ -99,7 +99,7 @@ func (ng *AlertNG) init() error {
|
||||
|
||||
decryptFn := ng.SecretsService.GetDecryptedValue
|
||||
multiOrgMetrics := ng.Metrics.GetMultiOrgAlertmanagerMetrics()
|
||||
ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, decryptFn, multiOrgMetrics, ng.NotificationService, log.New("ngalert.multiorg.alertmanager"))
|
||||
ng.MultiOrgAlertmanager, err = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore, decryptFn, multiOrgMetrics, ng.NotificationService, log.New("ngalert.multiorg.alertmanager"), ng.SecretsService)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
119
pkg/services/ngalert/notifier/alertmanager_config.go
Normal file
119
pkg/services/ngalert/notifier/alertmanager_config.go
Normal file
@ -0,0 +1,119 @@
|
||||
package notifier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/store"
|
||||
)
|
||||
|
||||
type UnknownReceiverError struct {
|
||||
UID string
|
||||
}
|
||||
|
||||
func (e UnknownReceiverError) Error() string {
|
||||
return fmt.Sprintf("unknown receiver: %s", e.UID)
|
||||
}
|
||||
|
||||
type AlertmanagerConfigRejectedError struct {
|
||||
Inner error
|
||||
}
|
||||
|
||||
func (e AlertmanagerConfigRejectedError) Error() string {
|
||||
return fmt.Sprintf("failed to save and apply Alertmanager configuration: %s", e.Inner.Error())
|
||||
}
|
||||
|
||||
type configurationStore interface {
|
||||
GetLatestAlertmanagerConfiguration(ctx context.Context, query *models.GetLatestAlertmanagerConfigurationQuery) error
|
||||
}
|
||||
|
||||
func (moa *MultiOrgAlertmanager) GetAlertmanagerConfiguration(ctx context.Context, org int64) (definitions.GettableUserConfig, error) {
|
||||
query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: org}
|
||||
err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, &query)
|
||||
if err != nil {
|
||||
return definitions.GettableUserConfig{}, fmt.Errorf("failed to get latest configuration: %w", err)
|
||||
}
|
||||
cfg, err := Load([]byte(query.Result.AlertmanagerConfiguration))
|
||||
if err != nil {
|
||||
return definitions.GettableUserConfig{}, fmt.Errorf("failed to unmarshal alertmanager configuration: %w", err)
|
||||
}
|
||||
|
||||
result := definitions.GettableUserConfig{
|
||||
TemplateFiles: cfg.TemplateFiles,
|
||||
AlertmanagerConfig: definitions.GettableApiAlertingConfig{
|
||||
Config: cfg.AlertmanagerConfig.Config,
|
||||
},
|
||||
}
|
||||
|
||||
for _, recv := range cfg.AlertmanagerConfig.Receivers {
|
||||
receivers := make([]*definitions.GettableGrafanaReceiver, 0, len(recv.PostableGrafanaReceivers.GrafanaManagedReceivers))
|
||||
for _, pr := range recv.PostableGrafanaReceivers.GrafanaManagedReceivers {
|
||||
secureFields := make(map[string]bool, len(pr.SecureSettings))
|
||||
for k := range pr.SecureSettings {
|
||||
decryptedValue, err := moa.Crypto.getDecryptedSecret(pr, k)
|
||||
if err != nil {
|
||||
return definitions.GettableUserConfig{}, fmt.Errorf("failed to decrypt stored secure setting: %w", err)
|
||||
}
|
||||
if decryptedValue == "" {
|
||||
continue
|
||||
}
|
||||
secureFields[k] = true
|
||||
}
|
||||
gr := definitions.GettableGrafanaReceiver{
|
||||
UID: pr.UID,
|
||||
Name: pr.Name,
|
||||
Type: pr.Type,
|
||||
DisableResolveMessage: pr.DisableResolveMessage,
|
||||
Settings: pr.Settings,
|
||||
SecureFields: secureFields,
|
||||
}
|
||||
receivers = append(receivers, &gr)
|
||||
}
|
||||
gettableApiReceiver := definitions.GettableApiReceiver{
|
||||
GettableGrafanaReceivers: definitions.GettableGrafanaReceivers{
|
||||
GrafanaManagedReceivers: receivers,
|
||||
},
|
||||
}
|
||||
gettableApiReceiver.Name = recv.Name
|
||||
result.AlertmanagerConfig.Receivers = append(result.AlertmanagerConfig.Receivers, &gettableApiReceiver)
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (moa *MultiOrgAlertmanager) ApplyAlertmanagerConfiguration(ctx context.Context, org int64, config definitions.PostableUserConfig) error {
|
||||
// Get the last known working configuration
|
||||
query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: org}
|
||||
if err := moa.configStore.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
|
||||
// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one
|
||||
if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
|
||||
return fmt.Errorf("failed to get latest configuration %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := moa.Crypto.LoadSecureSettings(ctx, org, config.AlertmanagerConfig.Receivers); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := config.ProcessConfig(moa.Crypto.Encrypt); err != nil {
|
||||
return fmt.Errorf("failed to post process Alertmanager configuration: %w", err)
|
||||
}
|
||||
|
||||
am, err := moa.AlertmanagerFor(org)
|
||||
if err != nil {
|
||||
// It's okay if the alertmanager isn't ready yet, we're changing its config anyway.
|
||||
if !errors.Is(err, ErrAlertmanagerNotReady) {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := am.SaveAndApplyConfig(ctx, &config); err != nil {
|
||||
moa.logger.Error("unable to save and apply alertmanager configuration", "err", err)
|
||||
return AlertmanagerConfigRejectedError{err}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
118
pkg/services/ngalert/notifier/crypto.go
Normal file
118
pkg/services/ngalert/notifier/crypto.go
Normal file
@ -0,0 +1,118 @@
|
||||
package notifier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/store"
|
||||
"github.com/grafana/grafana/pkg/services/secrets"
|
||||
)
|
||||
|
||||
// Crypto allows decryption of Alertmanager Configuration and encryption of arbitrary payloads.
|
||||
type Crypto interface {
|
||||
LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error
|
||||
Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error)
|
||||
|
||||
getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error)
|
||||
}
|
||||
|
||||
// alertmanagerCrypto implements decryption of Alertmanager configuration and encryption of arbitrary payloads based on Grafana's encryptions.
|
||||
type alertmanagerCrypto struct {
|
||||
secrets secrets.Service
|
||||
configs configurationStore
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func NewCrypto(secrets secrets.Service, configs configurationStore, log log.Logger) Crypto {
|
||||
return &alertmanagerCrypto{
|
||||
secrets: secrets,
|
||||
configs: configs,
|
||||
log: log,
|
||||
}
|
||||
}
|
||||
|
||||
// LoadSecureSettings adds the corresponding unencrypted secrets stored to the list of input receivers.
|
||||
func (c *alertmanagerCrypto) LoadSecureSettings(ctx context.Context, orgId int64, receivers []*definitions.PostableApiReceiver) error {
|
||||
// Get the last known working configuration.
|
||||
query := models.GetLatestAlertmanagerConfigurationQuery{OrgID: orgId}
|
||||
if err := c.configs.GetLatestAlertmanagerConfiguration(ctx, &query); err != nil {
|
||||
// If we don't have a configuration there's nothing for us to know and we should just continue saving the new one.
|
||||
if !errors.Is(err, store.ErrNoAlertmanagerConfiguration) {
|
||||
return fmt.Errorf("failed to get latest configuration: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
currentReceiverMap := make(map[string]*definitions.PostableGrafanaReceiver)
|
||||
if query.Result != nil {
|
||||
currentConfig, err := Load([]byte(query.Result.AlertmanagerConfiguration))
|
||||
// If the current config is un-loadable, treat it as if it never existed. Providing a new, valid config should be able to "fix" this state.
|
||||
if err != nil {
|
||||
c.log.Warn("Last known alertmanager configuration was invalid. Overwriting...")
|
||||
} else {
|
||||
currentReceiverMap = currentConfig.GetGrafanaReceiverMap()
|
||||
}
|
||||
}
|
||||
|
||||
// Copy the previously known secure settings.
|
||||
for i, r := range receivers {
|
||||
for j, gr := range r.PostableGrafanaReceivers.GrafanaManagedReceivers {
|
||||
if gr.UID == "" { // new receiver
|
||||
continue
|
||||
}
|
||||
|
||||
cgmr, ok := currentReceiverMap[gr.UID]
|
||||
if !ok {
|
||||
// It tries to update a receiver that didn't previously exist
|
||||
return UnknownReceiverError{UID: gr.UID}
|
||||
}
|
||||
|
||||
// Frontend sends only the secure settings that have to be updated
|
||||
// Therefore we have to copy from the last configuration only those secure settings not included in the request
|
||||
for key := range cgmr.SecureSettings {
|
||||
_, ok := gr.SecureSettings[key]
|
||||
if !ok {
|
||||
decryptedValue, err := c.getDecryptedSecret(cgmr, key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to decrypt stored secure setting: %s: %w", key, err)
|
||||
}
|
||||
|
||||
if receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings == nil {
|
||||
receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings = make(map[string]string, len(cgmr.SecureSettings))
|
||||
}
|
||||
|
||||
receivers[i].PostableGrafanaReceivers.GrafanaManagedReceivers[j].SecureSettings[key] = decryptedValue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *alertmanagerCrypto) getDecryptedSecret(r *definitions.PostableGrafanaReceiver, key string) (string, error) {
|
||||
storedValue, ok := r.SecureSettings[key]
|
||||
if !ok {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
decodeValue, err := base64.StdEncoding.DecodeString(storedValue)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
decryptedValue, err := c.secrets.Decrypt(context.Background(), decodeValue)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(decryptedValue), nil
|
||||
}
|
||||
|
||||
// Encrypt delegates encryption to secrets.Service.
|
||||
func (c *alertmanagerCrypto) Encrypt(ctx context.Context, payload []byte, opt secrets.EncryptionOptions) ([]byte, error) {
|
||||
return c.secrets.Encrypt(ctx, payload, opt)
|
||||
}
|
@ -11,6 +11,7 @@ import (
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/notifier/channels"
|
||||
"github.com/grafana/grafana/pkg/services/notifications"
|
||||
"github.com/grafana/grafana/pkg/services/secrets"
|
||||
|
||||
"github.com/prometheus/alertmanager/cluster"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
@ -29,6 +30,8 @@ var (
|
||||
)
|
||||
|
||||
type MultiOrgAlertmanager struct {
|
||||
Crypto Crypto
|
||||
|
||||
alertmanagersMtx sync.RWMutex
|
||||
alertmanagers map[int64]*Alertmanager
|
||||
|
||||
@ -51,9 +54,11 @@ type MultiOrgAlertmanager struct {
|
||||
|
||||
func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore,
|
||||
kvStore kvstore.KVStore, decryptFn channels.GetDecryptedValueFn, m *metrics.MultiOrgAlertmanager,
|
||||
ns notifications.Service, l log.Logger,
|
||||
ns notifications.Service, l log.Logger, s secrets.Service,
|
||||
) (*MultiOrgAlertmanager, error) {
|
||||
moa := &MultiOrgAlertmanager{
|
||||
Crypto: NewCrypto(s, configStore, l),
|
||||
|
||||
logger: l,
|
||||
settings: cfg,
|
||||
alertmanagers: map[int64]*Alertmanager{},
|
||||
|
@ -44,7 +44,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
|
||||
DisabledOrgs: map[int64]struct{}{5: {}},
|
||||
}, // do not poll in tests.
|
||||
}
|
||||
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
|
||||
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
|
||||
require.NoError(t, err)
|
||||
ctx := context.Background()
|
||||
|
||||
@ -168,7 +168,7 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgsWithFailures(t *testing.T)
|
||||
DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
|
||||
}, // do not poll in tests.
|
||||
}
|
||||
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
|
||||
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
|
||||
require.NoError(t, err)
|
||||
ctx := context.Background()
|
||||
|
||||
@ -218,7 +218,7 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
|
||||
decryptFn := secretsService.GetDecryptedValue
|
||||
reg := prometheus.NewPedanticRegistry()
|
||||
m := metrics.NewNGAlert(reg)
|
||||
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
|
||||
mam, err := NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
|
||||
require.NoError(t, err)
|
||||
ctx := context.Background()
|
||||
|
||||
|
@ -1015,7 +1015,7 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac
|
||||
m := metrics.NewNGAlert(registry)
|
||||
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
|
||||
decryptFn := secretsService.GetDecryptedValue
|
||||
moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, ¬ifier.FakeConfigStore{}, ¬ifier.FakeOrgStore{}, ¬ifier.FakeKVStore{}, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"))
|
||||
moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, ¬ifier.FakeConfigStore{}, ¬ifier.FakeOrgStore{}, ¬ifier.FakeKVStore{}, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
|
||||
require.NoError(t, err)
|
||||
|
||||
schedCfg := SchedulerCfg{
|
||||
|
Loading…
Reference in New Issue
Block a user