Alerting: take datasources as external alertmanagers into consideration (#52534)

This commit is contained in:
Jean-Philippe Quéméner 2022-07-20 16:50:49 +02:00 committed by GitHub
parent 5c4aa4a7ac
commit 50ae42130b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 332 additions and 24 deletions

View File

@ -49,6 +49,9 @@ func (s *FakeDataSourceService) GetAllDataSources(ctx context.Context, query *da
func (s *FakeDataSourceService) GetDataSourcesByType(ctx context.Context, query *datasources.GetDataSourcesByTypeQuery) error {
for _, datasource := range s.DataSources {
if query.OrgId > 0 && datasource.OrgId != query.OrgId {
continue
}
typeMatch := query.Type != "" && query.Type == datasource.Type
if typeMatch {
query.Result = append(query.Result, datasource)

View File

@ -165,6 +165,7 @@ type GetAllDataSourcesQuery struct {
}
type GetDataSourcesByTypeQuery struct {
OrgId int64 // optional: filter by org_id
Type string
Result []*DataSource
}

View File

@ -62,6 +62,7 @@ type AlertingStore interface {
type API struct {
Cfg *setting.Cfg
DatasourceCache datasources.CacheService
DatasourceService datasources.DataSourceService
RouteRegister routing.RouteRegister
ExpressionService *expr.Service
QuotaService quota.Service
@ -130,6 +131,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
}), m)
api.RegisterConfigurationApiEndpoints(NewConfiguration(
&ConfigSrv{
datasourceService: api.DatasourceService,
store: api.AdminConfigStore,
log: logger,
alertmanagerProvider: api.AlertsRouter,

View File

@ -1,12 +1,15 @@
package api
import (
"context"
"errors"
"fmt"
"net/http"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/datasources"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/store"
@ -16,6 +19,7 @@ import (
)
type ConfigSrv struct {
datasourceService datasources.DataSourceService
alertmanagerProvider ExternalAlertmanagerProvider
store store.AdminConfigurationStore
log log.Logger
@ -68,11 +72,17 @@ func (srv ConfigSrv) RoutePostNGalertConfig(c *models.ReqContext, body apimodels
sendAlertsTo, err := ngmodels.StringToAlertmanagersChoice(string(body.AlertmanagersChoice))
if err != nil {
return response.Error(400, "Invalid alertmanager choice specified", nil)
return response.Error(400, "Invalid alertmanager choice specified", err)
}
if sendAlertsTo == ngmodels.ExternalAlertmanagers && len(body.Alertmanagers) == 0 {
return response.Error(400, "At least one Alertmanager must be provided to choose this option", nil)
externalAlertmanagers, err := srv.externalAlertmanagers(c.Req.Context(), c.OrgId)
if err != nil {
return response.Error(500, "Couldn't fetch the external Alertmanagers from datasources", err)
}
if sendAlertsTo == ngmodels.ExternalAlertmanagers &&
len(body.Alertmanagers)+len(externalAlertmanagers) < 1 {
return response.Error(400, "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option", nil)
}
cfg := &ngmodels.AdminConfiguration{
@ -110,3 +120,25 @@ func (srv ConfigSrv) RouteDeleteNGalertConfig(c *models.ReqContext) response.Res
return response.JSON(http.StatusOK, util.DynMap{"message": "admin configuration deleted"})
}
// externalAlertmanagers returns the URL of any external alertmanager that is
// configured as datasource. The URL does not contain any auth.
func (srv ConfigSrv) externalAlertmanagers(ctx context.Context, orgID int64) ([]string, error) {
var alertmanagers []string
query := &datasources.GetDataSourcesByTypeQuery{
OrgId: orgID,
Type: datasources.DS_ALERTMANAGER,
}
err := srv.datasourceService.GetDataSourcesByType(ctx, query)
if err != nil {
return nil, fmt.Errorf("failed to fetch datasources for org: %w", err)
}
for _, ds := range query.Result {
if ds.JsonData.Get(apimodels.HandleGrafanaManagedAlerts).MustBool(false) {
// we don't need to build the exact URL as we only need
// to know if any is set
alertmanagers = append(alertmanagers, ds.Uid)
}
}
return alertmanagers, nil
}

View File

@ -0,0 +1,117 @@
package api
import (
"encoding/json"
"net/http"
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/datasources"
fakeDatasources "github.com/grafana/grafana/pkg/services/datasources/fakes"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/stretchr/testify/require"
)
func TestExternalAlertmanagerChoice(t *testing.T) {
tests := []struct {
name string
alertmanagerChoice definitions.AlertmanagersChoice
alertmanagers []string
datasources []*datasources.DataSource
statusCode int
message string
}{
{
name: "setting the choice to external by passing a plain url should succeed",
alertmanagerChoice: definitions.ExternalAlertmanagers,
alertmanagers: []string{"http://localhost:9000"},
datasources: []*datasources.DataSource{},
statusCode: http.StatusCreated,
message: "admin configuration updated",
},
{
name: "setting the choice to external by having a enabled external am datasource should succeed",
alertmanagerChoice: definitions.ExternalAlertmanagers,
alertmanagers: []string{},
datasources: []*datasources.DataSource{
{
OrgId: 1,
Type: datasources.DS_ALERTMANAGER,
Url: "http://localhost:9000",
JsonData: simplejson.NewFromAny(map[string]interface{}{
definitions.HandleGrafanaManagedAlerts: true,
}),
},
},
statusCode: http.StatusCreated,
message: "admin configuration updated",
},
{
name: "setting the choice to external by having a disabled external am datasource should fail",
alertmanagerChoice: definitions.ExternalAlertmanagers,
alertmanagers: []string{},
datasources: []*datasources.DataSource{
{
OrgId: 1,
Type: datasources.DS_ALERTMANAGER,
Url: "http://localhost:9000",
JsonData: simplejson.NewFromAny(map[string]interface{}{}),
},
},
statusCode: http.StatusBadRequest,
message: "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option",
},
{
name: "setting the choice to external and having no am configured should fail",
alertmanagerChoice: definitions.ExternalAlertmanagers,
alertmanagers: []string{},
datasources: []*datasources.DataSource{},
statusCode: http.StatusBadRequest,
message: "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option",
},
{
name: "setting the choice to all and having no external am configured should succeed",
alertmanagerChoice: definitions.AllAlertmanagers,
alertmanagers: []string{},
datasources: []*datasources.DataSource{},
statusCode: http.StatusCreated,
message: "admin configuration updated",
},
{
name: "setting the choice to internal should always succeed",
alertmanagerChoice: definitions.InternalAlertmanager,
alertmanagers: []string{},
datasources: []*datasources.DataSource{},
statusCode: http.StatusCreated,
message: "admin configuration updated",
},
}
ctx := createRequestCtxInOrg(1)
ctx.OrgRole = models.ROLE_ADMIN
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sut := createAPIAdminSut(t, test.datasources)
resp := sut.RoutePostNGalertConfig(ctx, definitions.PostableNGalertConfig{
Alertmanagers: test.alertmanagers,
AlertmanagersChoice: test.alertmanagerChoice,
})
var res map[string]interface{}
err := json.Unmarshal(resp.Body(), &res)
require.NoError(t, err)
require.Equal(t, test.message, res["message"])
require.Equal(t, test.statusCode, resp.Status())
})
}
}
func createAPIAdminSut(t *testing.T,
datasources []*datasources.DataSource) ConfigSrv {
return ConfigSrv{
datasourceService: &fakeDatasources.FakeDataSourceService{
DataSources: datasources,
},
store: store.NewFakeAdminConfigStore(t),
}
}

View File

@ -61,6 +61,7 @@ const (
AllAlertmanagers AlertmanagersChoice = "all"
InternalAlertmanager AlertmanagersChoice = "internal"
ExternalAlertmanagers AlertmanagersChoice = "external"
HandleGrafanaManagedAlerts = "handleGrafanaManagedAlerts"
)
// swagger:model

View File

@ -35,7 +35,7 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, routeRegister routing.RouteRegister,
func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, dataSourceService datasources.DataSourceService, routeRegister routing.RouteRegister,
sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, expressionService *expr.Service, dataProxy *datasourceproxy.DataSourceProxyService,
quotaService quota.Service, secretsService secrets.Service, notificationService notifications.Service, m *metrics.NGAlert,
folderService dashboards.FolderService, ac accesscontrol.AccessControl, dashboardService dashboards.DashboardService, renderService rendering.Service,
@ -43,6 +43,7 @@ func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService,
ng := &AlertNG{
Cfg: cfg,
DataSourceCache: dataSourceCache,
DataSourceService: dataSourceService,
RouteRegister: routeRegister,
SQLStore: sqlStore,
KVStore: kvStore,
@ -75,6 +76,7 @@ func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService,
type AlertNG struct {
Cfg *setting.Cfg
DataSourceCache datasources.CacheService
DataSourceService datasources.DataSourceService
RouteRegister routing.RouteRegister
SQLStore *sqlstore.SQLStore
KVStore kvstore.KVStore
@ -138,7 +140,8 @@ func (ng *AlertNG) init() error {
clk := clock.New()
alertsRouter := sender.NewAlertsRouter(ng.MultiOrgAlertmanager, store, clk, appUrl, ng.Cfg.UnifiedAlerting.DisabledOrgs, ng.Cfg.UnifiedAlerting.AdminConfigPollInterval)
alertsRouter := sender.NewAlertsRouter(ng.MultiOrgAlertmanager, store, clk, appUrl, ng.Cfg.UnifiedAlerting.DisabledOrgs,
ng.Cfg.UnifiedAlerting.AdminConfigPollInterval, ng.DataSourceService, ng.SecretsService)
// Make sure we sync at least once as Grafana starts to get the router up and running before we start sending any alerts.
if err := alertsRouter.SyncAndApplyConfigFromDatabase(); err != nil {
@ -176,6 +179,7 @@ func (ng *AlertNG) init() error {
api := api.API{
Cfg: ng.Cfg,
DatasourceCache: ng.DataSourceCache,
DatasourceService: ng.DataSourceService,
RouteRegister: ng.RouteRegister,
ExpressionService: ng.ExpressionService,
Schedule: ng.schedule,

View File

@ -3,6 +3,7 @@ package sender
import (
"context"
"errors"
"fmt"
"net/url"
"sync"
"time"
@ -10,10 +11,12 @@ import (
"github.com/benbjohnson/clock"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/services/secrets"
)
// AlertsRouter handles alerts generated during alert rule evaluation.
@ -38,9 +41,14 @@ type AlertsRouter struct {
appURL *url.URL
disabledOrgs map[int64]struct{}
adminConfigPollInterval time.Duration
datasourceService datasources.DataSourceService
secretService secrets.Service
}
func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore, clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration) *AlertsRouter {
func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore,
clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration,
datasourceService datasources.DataSourceService, secretService secrets.Service) *AlertsRouter {
d := &AlertsRouter{
logger: log.New("alerts-router"),
clock: clk,
@ -56,6 +64,9 @@ func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store stor
appURL: appURL,
disabledOrgs: disabledOrgs,
adminConfigPollInterval: configPollInterval,
datasourceService: datasourceService,
secretService: secretService,
}
return d
}
@ -87,17 +98,27 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
existing, ok := d.externalAlertmanagers[cfg.OrgID]
// We have no running sender and no Alertmanager(s) configured, no-op.
if !ok && len(cfg.Alertmanagers) == 0 {
d.logger.Debug("no external alertmanagers configured", "org", cfg.OrgID)
continue
}
// We have no running sender and alerts are handled internally, no-op.
if !ok && cfg.SendAlertsTo == models.InternalAlertmanager {
d.logger.Debug("alerts are handled internally", "org", cfg.OrgID)
continue
}
externalAlertmanagers, err := d.alertmanagersFromDatasources(cfg.OrgID)
if err != nil {
d.logger.Error("failed to get alertmanagers from datasources",
"org", cfg.OrgID,
"err", err)
continue
}
cfg.Alertmanagers = append(cfg.Alertmanagers, externalAlertmanagers...)
// We have no running sender and no Alertmanager(s) configured, no-op.
if !ok && len(cfg.Alertmanagers) == 0 {
d.logger.Debug("no external alertmanagers configured", "org", cfg.OrgID)
continue
}
// We have a running sender but no Alertmanager(s) configured, shut it down.
if ok && len(cfg.Alertmanagers) == 0 {
d.logger.Debug("no external alertmanager(s) configured, sender will be stopped", "org", cfg.OrgID)
@ -165,6 +186,56 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
return nil
}
func (d *AlertsRouter) alertmanagersFromDatasources(orgID int64) ([]string, error) {
var alertmanagers []string
// We might have alertmanager datasources that are acting as external
// alertmanager, let's fetch them.
query := &datasources.GetDataSourcesByTypeQuery{
OrgId: orgID,
Type: datasources.DS_ALERTMANAGER,
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
err := d.datasourceService.GetDataSourcesByType(ctx, query)
if err != nil {
return nil, fmt.Errorf("failed to fetch datasources for org: %w", err)
}
for _, ds := range query.Result {
if !ds.JsonData.Get(definitions.HandleGrafanaManagedAlerts).MustBool(false) {
continue
}
amURL, err := d.buildExternalURL(ds)
if err != nil {
d.logger.Error("failed to build external alertmanager URL",
"org", ds.OrgId,
"uid", ds.Uid,
"err", err)
continue
}
alertmanagers = append(alertmanagers, amURL)
}
return alertmanagers, nil
}
func (d *AlertsRouter) buildExternalURL(ds *datasources.DataSource) (string, error) {
amURL := ds.Url
// if basic auth is enabled we need to build the url with basic auth baked in
if !ds.BasicAuth {
return amURL, nil
}
parsed, err := url.Parse(ds.Url)
if err != nil {
return "", fmt.Errorf("failed to parse alertmanager datasource url: %w", err)
}
password := d.secretService.GetDecryptedValue(context.Background(), ds.SecureJsonData, "basicAuthPassword", "")
if password == "" {
return "", fmt.Errorf("basic auth enabled but no password set")
}
return fmt.Sprintf("%s://%s:%s@%s%s%s", parsed.Scheme, ds.BasicAuthUser,
password, parsed.Host, parsed.Path, parsed.RawQuery), nil
}
func (d *AlertsRouter) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) {
logger := d.logger.New("rule_uid", key.UID, "org", key.OrgID)
if len(alerts.PostableAlerts) == 0 {

View File

@ -15,13 +15,15 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/datasources"
fake_ds "github.com/grafana/grafana/pkg/services/datasources/fakes"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
fake_secrets "github.com/grafana/grafana/pkg/services/secrets/fakes"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
@ -45,7 +47,8 @@ func TestSendingToExternalAlertmanager(t *testing.T) {
Host: "localhost",
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute)
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
&fake_ds.FakeDataSourceService{}, fake_secrets.NewFakeSecretsService())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, Alertmanagers: []string{fakeAM.Server.URL}, SendAlertsTo: models.AllAlertmanagers},
@ -102,7 +105,8 @@ func TestSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) {
Host: "localhost",
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute)
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
&fake_ds.FakeDataSourceService{}, fake_secrets.NewFakeSecretsService())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey1.OrgID, Alertmanagers: []string{fakeAM.Server.URL}, SendAlertsTo: models.AllAlertmanagers},
@ -228,7 +232,8 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
Host: "localhost",
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute)
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{},
10*time.Minute, &fake_ds.FakeDataSourceService{}, fake_secrets.NewFakeSecretsService())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, Alertmanagers: []string{fakeAM.Server.URL}, SendAlertsTo: models.AllAlertmanagers},
@ -344,7 +349,7 @@ func createMultiOrgAlertmanager(t *testing.T, orgs []int64) *notifier.MultiOrgAl
kvStore := notifier.NewFakeKVStore(t)
registry := prometheus.NewPedanticRegistry()
m := metrics.NewNGAlert(registry)
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
secretsService := secretsManager.SetupTestService(t, fake_secrets.NewFakeSecretsStore())
decryptFn := secretsService.GetDecryptedValue
moa, err := notifier.NewMultiOrgAlertmanager(cfg, &cfgStore, &orgStore, kvStore, provisioning.NewFakeProvisioningStore(), decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
require.NoError(t, err)
@ -360,3 +365,60 @@ func createMultiOrgAlertmanager(t *testing.T, orgs []int64) *notifier.MultiOrgAl
}, 10*time.Second, 100*time.Millisecond)
return moa
}
func TestBuildExternalURL(t *testing.T) {
sch := AlertsRouter{
secretService: fake_secrets.NewFakeSecretsService(),
}
tests := []struct {
name string
ds *datasources.DataSource
expectedURL string
}{
{
name: "datasource without auth",
ds: &datasources.DataSource{
Url: "https://localhost:9000",
},
expectedURL: "https://localhost:9000",
},
{
name: "datasource without auth and with path",
ds: &datasources.DataSource{
Url: "https://localhost:9000/path/to/am",
},
expectedURL: "https://localhost:9000/path/to/am",
},
{
name: "datasource with auth",
ds: &datasources.DataSource{
Url: "https://localhost:9000",
BasicAuth: true,
BasicAuthUser: "johndoe",
SecureJsonData: map[string][]byte{
"basicAuthPassword": []byte("123"),
},
},
expectedURL: "https://johndoe:123@localhost:9000",
},
{
name: "datasource with auth and path",
ds: &datasources.DataSource{
Url: "https://localhost:9000/path/to/am",
BasicAuth: true,
BasicAuthUser: "johndoe",
SecureJsonData: map[string][]byte{
"basicAuthPassword": []byte("123"),
},
},
expectedURL: "https://johndoe:123@localhost:9000/path/to/am",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
url, err := sch.buildExternalURL(test.ds)
require.NoError(t, err)
require.Equal(t, test.expectedURL, url)
})
}
}

View File

@ -63,7 +63,7 @@ func SetupTestEnv(t *testing.T, baseInterval time.Duration) (*ngalert.AlertNG, *
)
ng, err := ngalert.ProvideService(
cfg, nil, routing.NewRouteRegister(), sqlStore, nil, nil, nil, nil,
cfg, nil, nil, routing.NewRouteRegister(), sqlStore, nil, nil, nil, nil,
secretsService, nil, m, folderService, ac, &dashboards.FakeDashboardService{}, nil, bus,
)
require.NoError(t, err)

View File

@ -76,6 +76,9 @@ func (ss *SQLStore) GetDataSourcesByType(ctx context.Context, query *datasources
query.Result = make([]*datasources.DataSource, 0)
return ss.WithDbSession(ctx, func(sess *DBSession) error {
if query.OrgId > 0 {
return sess.Where("type=? AND org_id=?", query.Type, query.OrgId).Asc("id").Find(&query.Result)
}
return sess.Where("type=?", query.Type).Asc("id").Find(&query.Result)
})
}

View File

@ -84,7 +84,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, `{"message": "Invalid alertmanager choice specified"}`, string(b))
var res map[string]interface{}
err = json.Unmarshal(b, &res)
require.NoError(t, err)
require.Equal(t, "Invalid alertmanager choice specified", res["message"])
}
// Let's try to send all the alerts to an external Alertmanager
@ -102,7 +105,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
resp := postRequest(t, alertsURL, buf.String(), http.StatusBadRequest) // nolint
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, `{"message": "At least one Alertmanager must be provided to choose this option"}`, string(b))
var res map[string]interface{}
err = json.Unmarshal(b, &res)
require.NoError(t, err)
require.Equal(t, "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option", res["message"])
}
// Now, lets re-set external Alertmanagers for main organisation
@ -121,7 +127,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
resp := postRequest(t, alertsURL, buf.String(), http.StatusCreated) // nolint
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, `{"message": "admin configuration updated"}`, string(b))
var res map[string]interface{}
err = json.Unmarshal(b, &res)
require.NoError(t, err)
require.Equal(t, "admin configuration updated", res["message"])
}
// If we get the configuration again, it shows us what we've set.
@ -220,7 +229,10 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
resp := postRequest(t, alertsURL, buf.String(), http.StatusCreated) // nolint
b, err := ioutil.ReadAll(resp.Body)
require.NoError(t, err)
require.JSONEq(t, "{\"message\": \"admin configuration updated\"}", string(b))
var res map[string]interface{}
err = json.Unmarshal(b, &res)
require.NoError(t, err)
require.Equal(t, "admin configuration updated", res["message"])
}
// If we get the configuration again, it shows us what we've set.