IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2024-11-25 10:20:29 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Implement SaveAndApplyConfig in the forked Alertmanager (remote primary) (#84659)

Browse Source 
* Alerting: Implement SaveAndApplyConfiguration in the forked Alertmanager struct

* call SaveAndApplyConfig on the remote first, log errors for the internal

* add comments explaining why we ignore errors in the internal AM

* restore go.work.sum
This commit is contained in:
Santiago 2024-04-23 15:45:35 +02:00 committed by GitHub
parent b28e6bc5d8
commit a6be12c037
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 33 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
pkg/services/ngalert/remote/forked_alertmanager_test.go
View File

@ -406,6 +406,26 @@ func TestForkedAlertmanager_ModeRemotePrimary(t *testing.T) {
require.NoError(tt, forked.SaveAndApplyDefaultConfig(ctx))
})
t.Run("SaveAndApplyConfig", func(tt *testing.T) {
// SaveAndApplyConfig should first be called on the remote Alertmanager
// and then on the internal one.
internal, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary)
remoteCall := remote.EXPECT().SaveAndApplyConfig(ctx, mock.Anything).Return(nil).Once()
internal.EXPECT().SaveAndApplyConfig(ctx, mock.Anything).Return(nil).Once().NotBefore(remoteCall)
require.NoError(tt, forked.SaveAndApplyConfig(ctx, &apimodels.PostableUserConfig{}))
// If there's an error in the remote Alertmanager, it should be returned.
_, remote, forked = genTestAlertmanagers(tt, modeRemotePrimary)
remote.EXPECT().SaveAndApplyConfig(ctx, mock.Anything).Return(expErr).Once()
require.ErrorIs(tt, expErr, forked.SaveAndApplyConfig(ctx, &apimodels.PostableUserConfig{}))
// An error in the internal Alertmanager should not be returned.
internal, remote, forked = genTestAlertmanagers(tt, modeRemotePrimary)
remote.EXPECT().SaveAndApplyConfig(ctx, mock.Anything).Return(nil).Once()
internal.EXPECT().SaveAndApplyConfig(ctx, mock.Anything).Return(expErr).Once()
require.NoError(tt, forked.SaveAndApplyConfig(ctx, &apimodels.PostableUserConfig{}))
})
t.Run("GetStatus", func(tt *testing.T) {
// We care about the status of the remote Alertmanager.
_, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary)

13
pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go
View File

@ -34,12 +34,23 @@ func (fam *RemotePrimaryForkedAlertmanager) ApplyConfig(ctx context.Context, con
}
if err := fam.internal.ApplyConfig(ctx, config); err != nil {
// An error in the internal Alertmanager shouldn't make the whole operation fail.
// We're replicating writes in the internal Alertmanager just for comparing and in case we need to roll back.
fam.log.Error("Error applying config to the internal Alertmanager", "err", err)
}
return nil
}
func (fam *RemotePrimaryForkedAlertmanager) SaveAndApplyConfig(ctx context.Context, config *apimodels.PostableUserConfig) error {
if err := fam.remote.SaveAndApplyConfig(ctx, config); err != nil {
return err
}
if err := fam.internal.SaveAndApplyConfig(ctx, config); err != nil {
// An error in the internal Alertmanager shouldn't make the whole operation fail.
// We're replicating writes in the internal Alertmanager just for comparing and in case we need to roll back.
fam.log.Error("Error applying config to the internal Alertmanager", "err", err)
}
return nil
}
@ -49,6 +60,8 @@ func (fam *RemotePrimaryForkedAlertmanager) SaveAndApplyDefaultConfig(ctx contex
}
if err := fam.internal.SaveAndApplyDefaultConfig(ctx); err != nil {
// An error in the internal Alertmanager shouldn't make the whole operation fail.
// We're replicating writes in the internal Alertmanager just for comparing and in case we need to roll back.
fam.log.Error("Error applying the default configuration to the internal Alertmanager", "err", err)
}
return nil