diff --git a/pkg/services/ngalert/api/api_alertmanager.go b/pkg/services/ngalert/api/api_alertmanager.go index 4e9debc356a..a508c02e3a2 100644 --- a/pkg/services/ngalert/api/api_alertmanager.go +++ b/pkg/services/ngalert/api/api_alertmanager.go @@ -48,7 +48,12 @@ func (srv AlertmanagerSrv) RouteGetAMStatus(c *contextmodel.ReqContext) response return errResp } - status := am.GetStatus() + status, err := am.GetStatus(c.Req.Context()) + if err != nil { + srv.log.Error("Unable to get status for the alertmanager", "error", err) + return ErrResp(http.StatusInternalServerError, err, "failed to get status for the Alertmanager") + } + if !c.SignedInUser.HasRole(org.RoleAdmin) { notifier.RemoveAutogenConfigIfExists(status.Config.Route) } diff --git a/pkg/services/ngalert/notifier/alertmanager_mock/Alertmanager.go b/pkg/services/ngalert/notifier/alertmanager_mock/Alertmanager.go index ce0916ef963..82d27daf97f 100644 --- a/pkg/services/ngalert/notifier/alertmanager_mock/Alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager_mock/Alertmanager.go @@ -422,22 +422,32 @@ func (_c *AlertmanagerMock_GetSilence_Call) RunAndReturn(run func(context.Contex return _c } -// GetStatus provides a mock function with given fields: -func (_m *AlertmanagerMock) GetStatus() definitions.GettableStatus { - ret := _m.Called() +// GetStatus provides a mock function with given fields: _a0 +func (_m *AlertmanagerMock) GetStatus(_a0 context.Context) (definitions.GettableStatus, error) { + ret := _m.Called(_a0) if len(ret) == 0 { panic("no return value specified for GetStatus") } var r0 definitions.GettableStatus - if rf, ok := ret.Get(0).(func() definitions.GettableStatus); ok { - r0 = rf() + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (definitions.GettableStatus, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(context.Context) definitions.GettableStatus); ok { + r0 = rf(_a0) } else { r0 = ret.Get(0).(definitions.GettableStatus) } - return r0 + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // AlertmanagerMock_GetStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetStatus' @@ -446,23 +456,24 @@ type AlertmanagerMock_GetStatus_Call struct { } // GetStatus is a helper method to define mock.On call -func (_e *AlertmanagerMock_Expecter) GetStatus() *AlertmanagerMock_GetStatus_Call { - return &AlertmanagerMock_GetStatus_Call{Call: _e.mock.On("GetStatus")} +// - _a0 context.Context +func (_e *AlertmanagerMock_Expecter) GetStatus(_a0 interface{}) *AlertmanagerMock_GetStatus_Call { + return &AlertmanagerMock_GetStatus_Call{Call: _e.mock.On("GetStatus", _a0)} } -func (_c *AlertmanagerMock_GetStatus_Call) Run(run func()) *AlertmanagerMock_GetStatus_Call { +func (_c *AlertmanagerMock_GetStatus_Call) Run(run func(_a0 context.Context)) *AlertmanagerMock_GetStatus_Call { _c.Call.Run(func(args mock.Arguments) { - run() + run(args[0].(context.Context)) }) return _c } -func (_c *AlertmanagerMock_GetStatus_Call) Return(_a0 definitions.GettableStatus) *AlertmanagerMock_GetStatus_Call { - _c.Call.Return(_a0) +func (_c *AlertmanagerMock_GetStatus_Call) Return(_a0 definitions.GettableStatus, _a1 error) *AlertmanagerMock_GetStatus_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *AlertmanagerMock_GetStatus_Call) RunAndReturn(run func() definitions.GettableStatus) *AlertmanagerMock_GetStatus_Call { +func (_c *AlertmanagerMock_GetStatus_Call) RunAndReturn(run func(context.Context) (definitions.GettableStatus, error)) *AlertmanagerMock_GetStatus_Call { _c.Call.Return(run) return _c } diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go index 120fb5e960d..80fa5dc155c 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go @@ -35,7 +35,7 @@ type Alertmanager interface { ApplyConfig(context.Context, *models.AlertConfiguration) error SaveAndApplyConfig(ctx context.Context, config *apimodels.PostableUserConfig) error SaveAndApplyDefaultConfig(ctx context.Context) error - GetStatus() apimodels.GettableStatus + GetStatus(context.Context) (apimodels.GettableStatus, error) // Silences CreateSilence(context.Context, *apimodels.PostableSilence) (string, error) diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go index ff88d51f17d..af5062533e3 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager_test.go @@ -217,7 +217,9 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) { require.NoError(t, err) internalAm, ok := am.(*alertmanager) require.True(t, ok) - require.Equal(t, "N/A", *am.GetStatus().VersionInfo.Version) + status, err := am.GetStatus(ctx) + require.NoError(t, err) + require.Equal(t, "N/A", *status.VersionInfo.Version) require.Equal(t, int64(2), internalAm.orgID) } diff --git a/pkg/services/ngalert/notifier/status.go b/pkg/services/ngalert/notifier/status.go index bcdf065616c..51e91563ef2 100644 --- a/pkg/services/ngalert/notifier/status.go +++ b/pkg/services/ngalert/notifier/status.go @@ -1,22 +1,24 @@ package notifier import ( + "context" "encoding/json" + "fmt" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" ) // TODO: We no longer do apimodels at this layer, move it to the API. -func (am *alertmanager) GetStatus() apimodels.GettableStatus { +func (am *alertmanager) GetStatus(_ context.Context) (apimodels.GettableStatus, error) { config := &apimodels.PostableUserConfig{} status := am.Base.GetStatus() // TODO: This should return a GettableStatus, for now it returns PostableUserConfig. if status == nil { - return *apimodels.NewGettableStatus(&config.AlertmanagerConfig) + return *apimodels.NewGettableStatus(&config.AlertmanagerConfig), nil } if err := json.Unmarshal(status, config); err != nil { - am.logger.Error("Unable to unmarshall alertmanager config", "Err", err) + return apimodels.GettableStatus{}, fmt.Errorf("unable to unmarshal alertmanager config: %w", err) } - return *apimodels.NewGettableStatus(&config.AlertmanagerConfig) + return *apimodels.NewGettableStatus(&config.AlertmanagerConfig), nil } diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go index 6862fba4e99..5631c9003f1 100644 --- a/pkg/services/ngalert/remote/alertmanager.go +++ b/pkg/services/ngalert/remote/alertmanager.go @@ -14,6 +14,7 @@ import ( "github.com/go-openapi/strfmt" amalert "github.com/prometheus/alertmanager/api/v2/client/alert" amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup" + amgeneral "github.com/prometheus/alertmanager/api/v2/client/general" amreceiver "github.com/prometheus/alertmanager/api/v2/client/receiver" amsilence "github.com/prometheus/alertmanager/api/v2/client/silence" @@ -27,6 +28,7 @@ import ( "github.com/grafana/grafana/pkg/services/ngalert/notifier" remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client" "github.com/grafana/grafana/pkg/services/ngalert/sender" + "gopkg.in/yaml.v3" ) type stateStore interface { @@ -421,8 +423,26 @@ func (am *Alertmanager) PutAlerts(ctx context.Context, alerts apimodels.Postable return nil } -func (am *Alertmanager) GetStatus() apimodels.GettableStatus { - return apimodels.GettableStatus{} +// GetStatus retrieves the remote Alertmanager configuration. +func (am *Alertmanager) GetStatus(ctx context.Context) (apimodels.GettableStatus, error) { + defer func() { + if r := recover(); r != nil { + am.log.Error("Panic while getting status", "err", r) + } + }() + + params := amgeneral.NewGetStatusParamsWithContext(ctx) + res, err := am.amClient.General.GetStatus(params) + if err != nil { + return apimodels.GettableStatus{}, err + } + + var cfg apimodels.PostableApiAlertingConfig + if err := yaml.Unmarshal([]byte(*res.Payload.Config.Original), &cfg); err != nil { + return apimodels.GettableStatus{}, err + } + + return *apimodels.NewGettableStatus(&cfg), nil } func (am *Alertmanager) GetReceivers(ctx context.Context) ([]apimodels.Receiver, error) { diff --git a/pkg/services/ngalert/remote/alertmanager_test.go b/pkg/services/ngalert/remote/alertmanager_test.go index 188889fd56b..fff6d177e0e 100644 --- a/pkg/services/ngalert/remote/alertmanager_test.go +++ b/pkg/services/ngalert/remote/alertmanager_test.go @@ -34,6 +34,7 @@ import ( "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/tests/testsuite" "github.com/grafana/grafana/pkg/util" + "gopkg.in/yaml.v3" ) const ( @@ -443,6 +444,39 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) { } } +func TestIntegrationRemoteAlertmanagerGetStatus(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + amURL, ok := os.LookupEnv("AM_URL") + if !ok { + t.Skip("No Alertmanager URL provided") + } + tenantID := os.Getenv("AM_TENANT_ID") + password := os.Getenv("AM_PASSWORD") + + cfg := AlertmanagerConfig{ + OrgID: 1, + URL: amURL, + TenantID: tenantID, + BasicAuthPassword: password, + } + + secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore()) + m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry()) + am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, defaultGrafanaConfig, m) + require.NoError(t, err) + + // We should get the default Cloud Alertmanager configuration. + ctx := context.Background() + status, err := am.GetStatus(ctx) + require.NoError(t, err) + b, err := yaml.Marshal(status.Config) + require.NoError(t, err) + require.YAMLEq(t, defaultCloudAMConfig, string(b)) +} + func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) { if testing.Short() { t.Skip("skipping integration test") @@ -640,3 +674,25 @@ func genAlert(active bool, labels map[string]string) amv2.PostableAlert { }, } } + +const defaultCloudAMConfig = ` +global: + resolve_timeout: 5m + http_config: + follow_redirects: true + enable_http2: true + smtp_hello: localhost + smtp_require_tls: true + pagerduty_url: https://events.pagerduty.com/v2/enqueue + opsgenie_api_url: https://api.opsgenie.com/ + wechat_api_url: https://qyapi.weixin.qq.com/cgi-bin/ + victorops_api_url: https://alert.victorops.com/integrations/generic/20131114/alert/ + telegram_api_url: https://api.telegram.org + webex_api_url: https://webexapis.com/v1/messages +route: + receiver: empty-receiver + continue: false +templates: [] +receivers: + - name: empty-receiver +` diff --git a/pkg/services/ngalert/remote/forked_alertmanager_test.go b/pkg/services/ngalert/remote/forked_alertmanager_test.go index 52a04116fbb..1d5914680ce 100644 --- a/pkg/services/ngalert/remote/forked_alertmanager_test.go +++ b/pkg/services/ngalert/remote/forked_alertmanager_test.go @@ -123,8 +123,10 @@ func TestForkedAlertmanager_ModeRemoteSecondary(t *testing.T) { // We care about the status of the internal Alertmanager. internal, _, forked := genTestAlertmanagers(tt, modeRemoteSecondary) status := apimodels.GettableStatus{} - internal.EXPECT().GetStatus().Return(status).Once() - require.Equal(tt, status, forked.GetStatus()) + internal.EXPECT().GetStatus(ctx).Return(status, nil).Once() + got, err := forked.GetStatus(ctx) + require.NoError(tt, err) + require.Equal(tt, status, got) }) t.Run("CreateSilence", func(tt *testing.T) { @@ -427,11 +429,23 @@ func TestForkedAlertmanager_ModeRemotePrimary(t *testing.T) { }) t.Run("GetStatus", func(tt *testing.T) { - // We care about the status of the remote Alertmanager. - _, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary) - status := apimodels.GettableStatus{} - remote.EXPECT().GetStatus().Return(status).Once() - require.Equal(tt, status, forked.GetStatus()) + { + // We care about the status of the remote Alertmanager. + _, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary) + status := apimodels.GettableStatus{} + remote.EXPECT().GetStatus(ctx).Return(status, nil).Once() + got, err := forked.GetStatus(ctx) + require.NoError(tt, err) + require.Equal(tt, status, got) + } + + { + // If there's an error in the remote Alertmanager, it should be returned. + _, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary) + remote.EXPECT().GetStatus(ctx).Return(apimodels.GettableStatus{}, expErr).Once() + _, err := forked.GetStatus(ctx) + require.ErrorIs(tt, expErr, err) + } }) t.Run("CreateSilence", func(tt *testing.T) { diff --git a/pkg/services/ngalert/remote/mock/remoteAlertmanager.go b/pkg/services/ngalert/remote/mock/remoteAlertmanager.go index 7e935f9cf7f..fac33bc02a6 100644 --- a/pkg/services/ngalert/remote/mock/remoteAlertmanager.go +++ b/pkg/services/ngalert/remote/mock/remoteAlertmanager.go @@ -515,22 +515,32 @@ func (_c *RemoteAlertmanagerMock_GetSilence_Call) RunAndReturn(run func(context. return _c } -// GetStatus provides a mock function with given fields: -func (_m *RemoteAlertmanagerMock) GetStatus() definitions.GettableStatus { - ret := _m.Called() +// GetStatus provides a mock function with given fields: _a0 +func (_m *RemoteAlertmanagerMock) GetStatus(_a0 context.Context) (definitions.GettableStatus, error) { + ret := _m.Called(_a0) if len(ret) == 0 { panic("no return value specified for GetStatus") } var r0 definitions.GettableStatus - if rf, ok := ret.Get(0).(func() definitions.GettableStatus); ok { - r0 = rf() + var r1 error + if rf, ok := ret.Get(0).(func(context.Context) (definitions.GettableStatus, error)); ok { + return rf(_a0) + } + if rf, ok := ret.Get(0).(func(context.Context) definitions.GettableStatus); ok { + r0 = rf(_a0) } else { r0 = ret.Get(0).(definitions.GettableStatus) } - return r0 + if rf, ok := ret.Get(1).(func(context.Context) error); ok { + r1 = rf(_a0) + } else { + r1 = ret.Error(1) + } + + return r0, r1 } // RemoteAlertmanagerMock_GetStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetStatus' @@ -539,23 +549,24 @@ type RemoteAlertmanagerMock_GetStatus_Call struct { } // GetStatus is a helper method to define mock.On call -func (_e *RemoteAlertmanagerMock_Expecter) GetStatus() *RemoteAlertmanagerMock_GetStatus_Call { - return &RemoteAlertmanagerMock_GetStatus_Call{Call: _e.mock.On("GetStatus")} +// - _a0 context.Context +func (_e *RemoteAlertmanagerMock_Expecter) GetStatus(_a0 interface{}) *RemoteAlertmanagerMock_GetStatus_Call { + return &RemoteAlertmanagerMock_GetStatus_Call{Call: _e.mock.On("GetStatus", _a0)} } -func (_c *RemoteAlertmanagerMock_GetStatus_Call) Run(run func()) *RemoteAlertmanagerMock_GetStatus_Call { +func (_c *RemoteAlertmanagerMock_GetStatus_Call) Run(run func(_a0 context.Context)) *RemoteAlertmanagerMock_GetStatus_Call { _c.Call.Run(func(args mock.Arguments) { - run() + run(args[0].(context.Context)) }) return _c } -func (_c *RemoteAlertmanagerMock_GetStatus_Call) Return(_a0 definitions.GettableStatus) *RemoteAlertmanagerMock_GetStatus_Call { - _c.Call.Return(_a0) +func (_c *RemoteAlertmanagerMock_GetStatus_Call) Return(_a0 definitions.GettableStatus, _a1 error) *RemoteAlertmanagerMock_GetStatus_Call { + _c.Call.Return(_a0, _a1) return _c } -func (_c *RemoteAlertmanagerMock_GetStatus_Call) RunAndReturn(run func() definitions.GettableStatus) *RemoteAlertmanagerMock_GetStatus_Call { +func (_c *RemoteAlertmanagerMock_GetStatus_Call) RunAndReturn(run func(context.Context) (definitions.GettableStatus, error)) *RemoteAlertmanagerMock_GetStatus_Call { _c.Call.Return(run) return _c } diff --git a/pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go b/pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go index 127d9d82e6f..8c4ef3b2400 100644 --- a/pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go +++ b/pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go @@ -67,8 +67,8 @@ func (fam *RemotePrimaryForkedAlertmanager) SaveAndApplyDefaultConfig(ctx contex return nil } -func (fam *RemotePrimaryForkedAlertmanager) GetStatus() apimodels.GettableStatus { - return fam.remote.GetStatus() +func (fam *RemotePrimaryForkedAlertmanager) GetStatus(ctx context.Context) (apimodels.GettableStatus, error) { + return fam.remote.GetStatus(ctx) } func (fam *RemotePrimaryForkedAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) { diff --git a/pkg/services/ngalert/remote/remote_secondary_forked_alertmanager.go b/pkg/services/ngalert/remote/remote_secondary_forked_alertmanager.go index dd96efdbb6a..5f9c9d3f641 100644 --- a/pkg/services/ngalert/remote/remote_secondary_forked_alertmanager.go +++ b/pkg/services/ngalert/remote/remote_secondary_forked_alertmanager.go @@ -123,8 +123,8 @@ func (fam *RemoteSecondaryForkedAlertmanager) SaveAndApplyDefaultConfig(ctx cont return fam.internal.SaveAndApplyDefaultConfig(ctx) } -func (fam *RemoteSecondaryForkedAlertmanager) GetStatus() apimodels.GettableStatus { - return fam.internal.GetStatus() +func (fam *RemoteSecondaryForkedAlertmanager) GetStatus(ctx context.Context) (apimodels.GettableStatus, error) { + return fam.internal.GetStatus(ctx) } func (fam *RemoteSecondaryForkedAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) {