Alerting: Implement GetStatus in the remote Alertmanager struct (#84887)

* Alerting: Implement GetStatus in the remote Alertmanager struct

* update tests

* fix tests, extract AlertmanagerConfig from PostableConfig

* get the remote AM config instead of the Grafana one from the remote AM

* pass grafana AM config in test

* return error in GetStatus instead of logging it (internal AM)

pkg/services/ngalert/api/api_alertmanager.go

@@ -48,7 +48,12 @@ func (srv AlertmanagerSrv) RouteGetAMStatus(c *contextmodel.ReqContext) response
 		return errResp
 	}
 
-	status := am.GetStatus()
+	status, err := am.GetStatus(c.Req.Context())
+	if err != nil {
+		srv.log.Error("Unable to get status for the alertmanager", "error", err)
+		return ErrResp(http.StatusInternalServerError, err, "failed to get status for the Alertmanager")
+	}
+
 	if !c.SignedInUser.HasRole(org.RoleAdmin) {
 		notifier.RemoveAutogenConfigIfExists(status.Config.Route)
 	}

pkg/services/ngalert/notifier/alertmanager_mock/Alertmanager.go

@@ -422,22 +422,32 @@ func (_c *AlertmanagerMock_GetSilence_Call) RunAndReturn(run func(context.Contex
 	return _c
 }
 
-// GetStatus provides a mock function with given fields:
+// GetStatus provides a mock function with given fields: _a0
-func (_m *AlertmanagerMock) GetStatus() definitions.GettableStatus {
+func (_m *AlertmanagerMock) GetStatus(_a0 context.Context) (definitions.GettableStatus, error) {
-	ret := _m.Called()
+	ret := _m.Called(_a0)
 
 	if len(ret) == 0 {
 		panic("no return value specified for GetStatus")
 	}
 
 	var r0 definitions.GettableStatus
-	if rf, ok := ret.Get(0).(func() definitions.GettableStatus); ok {
+	var r1 error
-		r0 = rf()
+	if rf, ok := ret.Get(0).(func(context.Context) (definitions.GettableStatus, error)); ok {
+		return rf(_a0)
+	}
+	if rf, ok := ret.Get(0).(func(context.Context) definitions.GettableStatus); ok {
+		r0 = rf(_a0)
 	} else {
 		r0 = ret.Get(0).(definitions.GettableStatus)
 	}
 
-	return r0
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
 }
 
 // AlertmanagerMock_GetStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetStatus'
@@ -446,23 +456,24 @@ type AlertmanagerMock_GetStatus_Call struct {
 }
 
 // GetStatus is a helper method to define mock.On call
-func (_e *AlertmanagerMock_Expecter) GetStatus() *AlertmanagerMock_GetStatus_Call {
+//   - _a0 context.Context
-	return &AlertmanagerMock_GetStatus_Call{Call: _e.mock.On("GetStatus")}
+func (_e *AlertmanagerMock_Expecter) GetStatus(_a0 interface{}) *AlertmanagerMock_GetStatus_Call {
+	return &AlertmanagerMock_GetStatus_Call{Call: _e.mock.On("GetStatus", _a0)}
 }
 
-func (_c *AlertmanagerMock_GetStatus_Call) Run(run func()) *AlertmanagerMock_GetStatus_Call {
+func (_c *AlertmanagerMock_GetStatus_Call) Run(run func(_a0 context.Context)) *AlertmanagerMock_GetStatus_Call {
 	_c.Call.Run(func(args mock.Arguments) {
-		run()
+		run(args[0].(context.Context))
 	})
 	return _c
 }
 
-func (_c *AlertmanagerMock_GetStatus_Call) Return(_a0 definitions.GettableStatus) *AlertmanagerMock_GetStatus_Call {
+func (_c *AlertmanagerMock_GetStatus_Call) Return(_a0 definitions.GettableStatus, _a1 error) *AlertmanagerMock_GetStatus_Call {
-	_c.Call.Return(_a0)
+	_c.Call.Return(_a0, _a1)
 	return _c
 }
 
-func (_c *AlertmanagerMock_GetStatus_Call) RunAndReturn(run func() definitions.GettableStatus) *AlertmanagerMock_GetStatus_Call {
+func (_c *AlertmanagerMock_GetStatus_Call) RunAndReturn(run func(context.Context) (definitions.GettableStatus, error)) *AlertmanagerMock_GetStatus_Call {
 	_c.Call.Return(run)
 	return _c
 }

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -35,7 +35,7 @@ type Alertmanager interface {
 	ApplyConfig(context.Context, *models.AlertConfiguration) error
 	SaveAndApplyConfig(ctx context.Context, config *apimodels.PostableUserConfig) error
 	SaveAndApplyDefaultConfig(ctx context.Context) error
-	GetStatus() apimodels.GettableStatus
+	GetStatus(context.Context) (apimodels.GettableStatus, error)
 
 	// Silences
 	CreateSilence(context.Context, *apimodels.PostableSilence) (string, error)

pkg/services/ngalert/notifier/multiorg_alertmanager_test.go

@@ -217,7 +217,9 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
 		require.NoError(t, err)
 		internalAm, ok := am.(*alertmanager)
 		require.True(t, ok)
-		require.Equal(t, "N/A", *am.GetStatus().VersionInfo.Version)
+		status, err := am.GetStatus(ctx)
+		require.NoError(t, err)
+		require.Equal(t, "N/A", *status.VersionInfo.Version)
 		require.Equal(t, int64(2), internalAm.orgID)
 	}
 

pkg/services/ngalert/notifier/status.go

@@ -1,22 +1,24 @@
 package notifier
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 )
 
 // TODO: We no longer do apimodels at this layer, move it to the API.
-func (am *alertmanager) GetStatus() apimodels.GettableStatus {
+func (am *alertmanager) GetStatus(_ context.Context) (apimodels.GettableStatus, error) {
 	config := &apimodels.PostableUserConfig{}
 	status := am.Base.GetStatus() // TODO: This should return a GettableStatus, for now it returns PostableUserConfig.
 	if status == nil {
-		return *apimodels.NewGettableStatus(&config.AlertmanagerConfig)
+		return *apimodels.NewGettableStatus(&config.AlertmanagerConfig), nil
 	}
 
 	if err := json.Unmarshal(status, config); err != nil {
-		am.logger.Error("Unable to unmarshall alertmanager config", "Err", err)
+		return apimodels.GettableStatus{}, fmt.Errorf("unable to unmarshal alertmanager config: %w", err)
 	}
 
-	return *apimodels.NewGettableStatus(&config.AlertmanagerConfig)
+	return *apimodels.NewGettableStatus(&config.AlertmanagerConfig), nil
 }

pkg/services/ngalert/remote/alertmanager.go

@@ -14,6 +14,7 @@ import (
 	"github.com/go-openapi/strfmt"
 	amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
 	amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup"
+	amgeneral "github.com/prometheus/alertmanager/api/v2/client/general"
 	amreceiver "github.com/prometheus/alertmanager/api/v2/client/receiver"
 	amsilence "github.com/prometheus/alertmanager/api/v2/client/silence"
 
@@ -27,6 +28,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client"
 	"github.com/grafana/grafana/pkg/services/ngalert/sender"
+	"gopkg.in/yaml.v3"
 )
 
 type stateStore interface {
@@ -421,8 +423,26 @@ func (am *Alertmanager) PutAlerts(ctx context.Context, alerts apimodels.Postable
 	return nil
 }
 
-func (am *Alertmanager) GetStatus() apimodels.GettableStatus {
+// GetStatus retrieves the remote Alertmanager configuration.
-	return apimodels.GettableStatus{}
+func (am *Alertmanager) GetStatus(ctx context.Context) (apimodels.GettableStatus, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			am.log.Error("Panic while getting status", "err", r)
+		}
+	}()
+
+	params := amgeneral.NewGetStatusParamsWithContext(ctx)
+	res, err := am.amClient.General.GetStatus(params)
+	if err != nil {
+		return apimodels.GettableStatus{}, err
+	}
+
+	var cfg apimodels.PostableApiAlertingConfig
+	if err := yaml.Unmarshal([]byte(*res.Payload.Config.Original), &cfg); err != nil {
+		return apimodels.GettableStatus{}, err
+	}
+
+	return *apimodels.NewGettableStatus(&cfg), nil
 }
 
 func (am *Alertmanager) GetReceivers(ctx context.Context) ([]apimodels.Receiver, error) {

pkg/services/ngalert/remote/alertmanager_test.go

@@ -34,6 +34,7 @@ import (
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tests/testsuite"
 	"github.com/grafana/grafana/pkg/util"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -443,6 +444,39 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
 	}
 }
 
+func TestIntegrationRemoteAlertmanagerGetStatus(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	amURL, ok := os.LookupEnv("AM_URL")
+	if !ok {
+		t.Skip("No Alertmanager URL provided")
+	}
+	tenantID := os.Getenv("AM_TENANT_ID")
+	password := os.Getenv("AM_PASSWORD")
+
+	cfg := AlertmanagerConfig{
+		OrgID:             1,
+		URL:               amURL,
+		TenantID:          tenantID,
+		BasicAuthPassword: password,
+	}
+
+	secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
+	m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
+	am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, defaultGrafanaConfig, m)
+	require.NoError(t, err)
+
+	// We should get the default Cloud Alertmanager configuration.
+	ctx := context.Background()
+	status, err := am.GetStatus(ctx)
+	require.NoError(t, err)
+	b, err := yaml.Marshal(status.Config)
+	require.NoError(t, err)
+	require.YAMLEq(t, defaultCloudAMConfig, string(b))
+}
+
 func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
 	if testing.Short() {
 		t.Skip("skipping integration test")
@@ -640,3 +674,25 @@ func genAlert(active bool, labels map[string]string) amv2.PostableAlert {
 		},
 	}
 }
+
+const defaultCloudAMConfig = `
+global:
+    resolve_timeout: 5m
+    http_config:
+        follow_redirects: true
+        enable_http2: true
+    smtp_hello: localhost
+    smtp_require_tls: true
+    pagerduty_url: https://events.pagerduty.com/v2/enqueue
+    opsgenie_api_url: https://api.opsgenie.com/
+    wechat_api_url: https://qyapi.weixin.qq.com/cgi-bin/
+    victorops_api_url: https://alert.victorops.com/integrations/generic/20131114/alert/
+    telegram_api_url: https://api.telegram.org
+    webex_api_url: https://webexapis.com/v1/messages
+route:
+    receiver: empty-receiver
+    continue: false
+templates: []
+receivers:
+    - name: empty-receiver
+`

pkg/services/ngalert/remote/forked_alertmanager_test.go

@@ -123,8 +123,10 @@ func TestForkedAlertmanager_ModeRemoteSecondary(t *testing.T) {
 		// We care about the status of the internal Alertmanager.
 		internal, _, forked := genTestAlertmanagers(tt, modeRemoteSecondary)
 		status := apimodels.GettableStatus{}
-		internal.EXPECT().GetStatus().Return(status).Once()
+		internal.EXPECT().GetStatus(ctx).Return(status, nil).Once()
-		require.Equal(tt, status, forked.GetStatus())
+		got, err := forked.GetStatus(ctx)
+		require.NoError(tt, err)
+		require.Equal(tt, status, got)
 	})
 
 	t.Run("CreateSilence", func(tt *testing.T) {
@@ -427,11 +429,23 @@ func TestForkedAlertmanager_ModeRemotePrimary(t *testing.T) {
 	})
 
 	t.Run("GetStatus", func(tt *testing.T) {
+		{
 			// We care about the status of the remote Alertmanager.
 			_, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary)
 			status := apimodels.GettableStatus{}
-		remote.EXPECT().GetStatus().Return(status).Once()
+			remote.EXPECT().GetStatus(ctx).Return(status, nil).Once()
-		require.Equal(tt, status, forked.GetStatus())
+			got, err := forked.GetStatus(ctx)
+			require.NoError(tt, err)
+			require.Equal(tt, status, got)
+		}
+
+		{
+			// If there's an error in the remote Alertmanager, it should be returned.
+			_, remote, forked := genTestAlertmanagers(tt, modeRemotePrimary)
+			remote.EXPECT().GetStatus(ctx).Return(apimodels.GettableStatus{}, expErr).Once()
+			_, err := forked.GetStatus(ctx)
+			require.ErrorIs(tt, expErr, err)
+		}
 	})
 
 	t.Run("CreateSilence", func(tt *testing.T) {

pkg/services/ngalert/remote/mock/remoteAlertmanager.go

@@ -515,22 +515,32 @@ func (_c *RemoteAlertmanagerMock_GetSilence_Call) RunAndReturn(run func(context.
 	return _c
 }
 
-// GetStatus provides a mock function with given fields:
+// GetStatus provides a mock function with given fields: _a0
-func (_m *RemoteAlertmanagerMock) GetStatus() definitions.GettableStatus {
+func (_m *RemoteAlertmanagerMock) GetStatus(_a0 context.Context) (definitions.GettableStatus, error) {
-	ret := _m.Called()
+	ret := _m.Called(_a0)
 
 	if len(ret) == 0 {
 		panic("no return value specified for GetStatus")
 	}
 
 	var r0 definitions.GettableStatus
-	if rf, ok := ret.Get(0).(func() definitions.GettableStatus); ok {
+	var r1 error
-		r0 = rf()
+	if rf, ok := ret.Get(0).(func(context.Context) (definitions.GettableStatus, error)); ok {
+		return rf(_a0)
+	}
+	if rf, ok := ret.Get(0).(func(context.Context) definitions.GettableStatus); ok {
+		r0 = rf(_a0)
 	} else {
 		r0 = ret.Get(0).(definitions.GettableStatus)
 	}
 
-	return r0
+	if rf, ok := ret.Get(1).(func(context.Context) error); ok {
+		r1 = rf(_a0)
+	} else {
+		r1 = ret.Error(1)
+	}
+
+	return r0, r1
 }
 
 // RemoteAlertmanagerMock_GetStatus_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'GetStatus'
@@ -539,23 +549,24 @@ type RemoteAlertmanagerMock_GetStatus_Call struct {
 }
 
 // GetStatus is a helper method to define mock.On call
-func (_e *RemoteAlertmanagerMock_Expecter) GetStatus() *RemoteAlertmanagerMock_GetStatus_Call {
+//   - _a0 context.Context
-	return &RemoteAlertmanagerMock_GetStatus_Call{Call: _e.mock.On("GetStatus")}
+func (_e *RemoteAlertmanagerMock_Expecter) GetStatus(_a0 interface{}) *RemoteAlertmanagerMock_GetStatus_Call {
+	return &RemoteAlertmanagerMock_GetStatus_Call{Call: _e.mock.On("GetStatus", _a0)}
 }
 
-func (_c *RemoteAlertmanagerMock_GetStatus_Call) Run(run func()) *RemoteAlertmanagerMock_GetStatus_Call {
+func (_c *RemoteAlertmanagerMock_GetStatus_Call) Run(run func(_a0 context.Context)) *RemoteAlertmanagerMock_GetStatus_Call {
 	_c.Call.Run(func(args mock.Arguments) {
-		run()
+		run(args[0].(context.Context))
 	})
 	return _c
 }
 
-func (_c *RemoteAlertmanagerMock_GetStatus_Call) Return(_a0 definitions.GettableStatus) *RemoteAlertmanagerMock_GetStatus_Call {
+func (_c *RemoteAlertmanagerMock_GetStatus_Call) Return(_a0 definitions.GettableStatus, _a1 error) *RemoteAlertmanagerMock_GetStatus_Call {
-	_c.Call.Return(_a0)
+	_c.Call.Return(_a0, _a1)
 	return _c
 }
 
-func (_c *RemoteAlertmanagerMock_GetStatus_Call) RunAndReturn(run func() definitions.GettableStatus) *RemoteAlertmanagerMock_GetStatus_Call {
+func (_c *RemoteAlertmanagerMock_GetStatus_Call) RunAndReturn(run func(context.Context) (definitions.GettableStatus, error)) *RemoteAlertmanagerMock_GetStatus_Call {
 	_c.Call.Return(run)
 	return _c
 }

pkg/services/ngalert/remote/remote_primary_forked_alertmanager.go

@@ -67,8 +67,8 @@ func (fam *RemotePrimaryForkedAlertmanager) SaveAndApplyDefaultConfig(ctx contex
 	return nil
 }
 
-func (fam *RemotePrimaryForkedAlertmanager) GetStatus() apimodels.GettableStatus {
+func (fam *RemotePrimaryForkedAlertmanager) GetStatus(ctx context.Context) (apimodels.GettableStatus, error) {
-	return fam.remote.GetStatus()
+	return fam.remote.GetStatus(ctx)
 }
 
 func (fam *RemotePrimaryForkedAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) {

pkg/services/ngalert/remote/remote_secondary_forked_alertmanager.go

@@ -123,8 +123,8 @@ func (fam *RemoteSecondaryForkedAlertmanager) SaveAndApplyDefaultConfig(ctx cont
 	return fam.internal.SaveAndApplyDefaultConfig(ctx)
 }
 
-func (fam *RemoteSecondaryForkedAlertmanager) GetStatus() apimodels.GettableStatus {
+func (fam *RemoteSecondaryForkedAlertmanager) GetStatus(ctx context.Context) (apimodels.GettableStatus, error) {
-	return fam.internal.GetStatus()
+	return fam.internal.GetStatus(ctx)
 }
 
 func (fam *RemoteSecondaryForkedAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) {