IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Feature toggle to disallow sending alerts externally (#87982)

Browse Source 
* Define feature toggle

* Implement feature toggle
This commit is contained in:
Steve Simpson 2024-05-23 14:29:19 +02:00 committed by GitHub
parent bd2b248f0e
commit 8421919cb5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 2170 additions and 2036 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
packages/grafana-data/src/types/featureToggles.gen.ts
View File

@ -190,4 +190,5 @@ export interface FeatureToggles {
notificationBanner?: boolean;
dashboardRestore?: boolean;
datasourceProxyDisableRBAC?: boolean;
alertingDisableSendAlertsExternal?: boolean;
}

9
pkg/services/featuremgmt/registry.go
View File

@ -1282,6 +1282,15 @@ var (
Owner: identityAccessTeam,
HideFromDocs: true,
},
{
Name: "alertingDisableSendAlertsExternal",
Description: "Disables the ability to send alerts to an external Alertmanager datasource.",
Stage: FeatureStageExperimental,
Owner: grafanaAlertingSquad,
AllowSelfServe: false,
HideFromDocs: true,
HideFromAdminPage: true,
},
}
)

1
pkg/services/featuremgmt/toggles_gen.csv
View File

@ -171,3 +171,4 @@ newDashboardSharingComponent,experimental,@grafana/sharing-squad,false,false,tru
notificationBanner,experimental,@grafana/grafana-frontend-platform,false,false,false
dashboardRestore,experimental,@grafana/grafana-frontend-platform,false,false,false
datasourceProxyDisableRBAC,GA,@grafana/identity-access-team,false,false,false
alertingDisableSendAlertsExternal,experimental,@grafana/alerting-squad,false,false,false

1 Name Stage Owner requiresDevMode RequiresRestart FrontendOnly
171 notificationBanner experimental @grafana/grafana-frontend-platform false false false
172 dashboardRestore experimental @grafana/grafana-frontend-platform false false false
173 datasourceProxyDisableRBAC GA @grafana/identity-access-team false false false
174 alertingDisableSendAlertsExternal experimental @grafana/alerting-squad false false false

4
pkg/services/featuremgmt/toggles_gen.go
View File

@ -694,4 +694,8 @@ const (
// FlagDatasourceProxyDisableRBAC
// Disables applying a plugin route's ReqAction field to authorization
FlagDatasourceProxyDisableRBAC = "datasourceProxyDisableRBAC"
// FlagAlertingDisableSendAlertsExternal
// Disables the ability to send alerts to an external Alertmanager datasource.
FlagAlertingDisableSendAlertsExternal = "alertingDisableSendAlertsExternal"
)

4017
pkg/services/featuremgmt/toggles_gen.json
View File

File diff suppressed because it is too large Load Diff

1
pkg/services/ngalert/api/api.go
View File

@ -144,6 +144,7 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
store: api.AdminConfigStore,
log: logger,
alertmanagerProvider: api.AlertsRouter,
featureManager: api.FeatureManager,
},
), m)

7
pkg/services/ngalert/api/api_configuration.go
View File

@ -12,6 +12,7 @@ import (
"github.com/grafana/grafana/pkg/infra/log"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/store"
@ -24,6 +25,7 @@ type ConfigSrv struct {
alertmanagerProvider ExternalAlertmanagerProvider
store store.AdminConfigurationStore
log log.Logger
featureManager featuremgmt.FeatureToggles
}
func (srv ConfigSrv) RouteGetAlertmanagers(c *contextmodel.ReqContext) response.Response {
@ -75,6 +77,11 @@ func (srv ConfigSrv) RoutePostNGalertConfig(c *contextmodel.ReqContext, body api
return response.Error(http.StatusBadRequest, "Invalid alertmanager choice specified", err)
}
disableExternal := srv.featureManager.IsEnabled(c.Req.Context(), featuremgmt.FlagAlertingDisableSendAlertsExternal)
if disableExternal && sendAlertsTo != ngmodels.InternalAlertmanager {
return response.Error(http.StatusBadRequest, "Sending alerts to external alertmanagers is disallowed on this instance", err)
}
externalAlertmanagers, err := srv.externalAlertmanagers(c.Req.Context(), c.SignedInUser.GetOrgID())
if err != nil {
return response.Error(http.StatusInternalServerError, "Couldn't fetch the external Alertmanagers from datasources", err)

36
pkg/services/ngalert/api/api_configuration_test.go
View File

@ -10,6 +10,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/services/datasources"
fakeDatasources "github.com/grafana/grafana/pkg/services/datasources/fakes"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/store"
"github.com/grafana/grafana/pkg/services/org"
@ -22,6 +23,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
datasources []*datasources.DataSource
statusCode int
message string
features featuremgmt.FeatureToggles
}{
{
name: "setting the choice to external by having a enabled external am datasource should succeed",
@ -38,6 +40,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
},
statusCode: http.StatusCreated,
message: "admin configuration updated",
features: featuremgmt.WithFeatures(),
},
{
name: "setting the choice to external by having a disabled external am datasource should fail",
@ -52,6 +55,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
},
statusCode: http.StatusBadRequest,
message: "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option",
features: featuremgmt.WithFeatures(),
},
{
name: "setting the choice to external and having no am configured should fail",
@ -59,6 +63,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
datasources: []*datasources.DataSource{},
statusCode: http.StatusBadRequest,
message: "At least one Alertmanager must be provided or configured as a datasource that handles alerts to choose this option",
features: featuremgmt.WithFeatures(),
},
{
name: "setting the choice to all and having no external am configured should succeed",
@ -66,6 +71,7 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
datasources: []*datasources.DataSource{},
statusCode: http.StatusCreated,
message: "admin configuration updated",
features: featuremgmt.WithFeatures(),
},
{
name: "setting the choice to internal should always succeed",
@ -73,13 +79,38 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
datasources: []*datasources.DataSource{},
statusCode: http.StatusCreated,
message: "admin configuration updated",
features: featuremgmt.WithFeatures(),
},
{
name: "setting the choice to internal should succeed when external disallowed",
alertmanagerChoice: definitions.InternalAlertmanager,
datasources: []*datasources.DataSource{},
statusCode: http.StatusCreated,
message: "admin configuration updated",
features: featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal),
},
{
name: "setting the choice to all should fail when external disallowed",
alertmanagerChoice: definitions.AllAlertmanagers,
datasources: []*datasources.DataSource{},
statusCode: http.StatusBadRequest,
message: "Sending alerts to external alertmanagers is disallowed on this instance",
features: featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal),
},
{
name: "setting the choice to external should fail when external disallowed",
alertmanagerChoice: definitions.ExternalAlertmanagers,
datasources: []*datasources.DataSource{},
statusCode: http.StatusBadRequest,
message: "Sending alerts to external alertmanagers is disallowed on this instance",
features: featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal),
},
}
ctx := createRequestCtxInOrg(1)
ctx.OrgRole = org.RoleAdmin
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
sut := createAPIAdminSut(t, test.datasources)
sut := createAPIAdminSut(t, test.datasources, test.features)
resp := sut.RoutePostNGalertConfig(ctx, definitions.PostableNGalertConfig{
AlertmanagersChoice: test.alertmanagerChoice,
})
@ -93,11 +124,12 @@ func TestExternalAlertmanagerChoice(t *testing.T) {
}
func createAPIAdminSut(t *testing.T,
datasources []*datasources.DataSource) ConfigSrv {
datasources []*datasources.DataSource, features featuremgmt.FeatureToggles) ConfigSrv {
return ConfigSrv{
datasourceService: &fakeDatasources.FakeDataSourceService{
DataSources: datasources,
},
store: store.NewFakeAdminConfigStore(t),
featureManager: features,
}
}

4
pkg/services/ngalert/ngalert.go
View File

@ -250,10 +250,10 @@ func (ng *AlertNG) init() error {
clk := clock.New()
alertsRouter := sender.NewAlertsRouter(ng.MultiOrgAlertmanager, ng.store, clk, appUrl, ng.Cfg.UnifiedAlerting.DisabledOrgs,
ng.Cfg.UnifiedAlerting.AdminConfigPollInterval, ng.DataSourceService, ng.SecretsService)
ng.Cfg.UnifiedAlerting.AdminConfigPollInterval, ng.DataSourceService, ng.SecretsService, ng.FeatureToggles)
// Make sure we sync at least once as Grafana starts to get the router up and running before we start sending any alerts.
if err := alertsRouter.SyncAndApplyConfigFromDatabase(); err != nil {
if err := alertsRouter.SyncAndApplyConfigFromDatabase(initCtx); err != nil {
return fmt.Errorf("failed to initialize alerting because alert notifications router failed to warm up: %w", err)
}

16
pkg/services/ngalert/sender/router.go
View File

@ -16,6 +16,7 @@ import (
"github.com/grafana/grafana/pkg/api/datasource"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/datasources"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
@ -48,11 +49,12 @@ type AlertsRouter struct {
datasourceService datasources.DataSourceService
secretService secrets.Service
featureManager featuremgmt.FeatureToggles
}
func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store store.AdminConfigurationStore,
clk clock.Clock, appURL *url.URL, disabledOrgs map[int64]struct{}, configPollInterval time.Duration,
datasourceService datasources.DataSourceService, secretService secrets.Service) *AlertsRouter {
datasourceService datasources.DataSourceService, secretService secrets.Service, featureManager featuremgmt.FeatureToggles) *AlertsRouter {
d := &AlertsRouter{
logger: log.New("ngalert.sender.router"),
clock: clk,
@ -71,13 +73,14 @@ func NewAlertsRouter(multiOrgNotifier *notifier.MultiOrgAlertmanager, store stor
datasourceService: datasourceService,
secretService: secretService,
featureManager: featureManager,
}
return d
}
// SyncAndApplyConfigFromDatabase looks for the admin configuration in the database
// and adjusts the sender(s) and alert handling mechanism accordingly.
func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
func (d *AlertsRouter) SyncAndApplyConfigFromDatabase(ctx context.Context) error {
cfgs, err := d.adminConfigStore.GetAdminConfigurations()
if err != nil {
return err
@ -85,6 +88,8 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
d.logger.Debug("Attempting to sync admin configs", "count", len(cfgs))
disableExternal := d.featureManager.IsEnabled(ctx, featuremgmt.FlagAlertingDisableSendAlertsExternal)
orgsFound := make(map[int64]struct{}, len(cfgs))
d.adminConfigMtx.Lock()
for _, cfg := range cfgs {
@ -93,6 +98,11 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
continue
}
if disableExternal && cfg.SendAlertsTo != models.InternalAlertmanager {
d.logger.Warn("Alertmanager choice in configuration will be ignored due to feature flags", "org", cfg.OrgID, "choice", cfg.SendAlertsTo)
cfg.SendAlertsTo = models.InternalAlertmanager
}
// Update the Alertmanagers choice for the organization.
d.sendAlertsTo[cfg.OrgID] = cfg.SendAlertsTo
@ -363,7 +373,7 @@ func (d *AlertsRouter) Run(ctx context.Context) error {
for {
select {
case <-time.After(d.adminConfigPollInterval):
if err := d.SyncAndApplyConfigFromDatabase(); err != nil {
if err := d.SyncAndApplyConfigFromDatabase(ctx); err != nil {
d.logger.Error("Unable to sync admin configuration", "error", err)
}
case <-ctx.Done():

108
pkg/services/ngalert/sender/router_test.go
View File

@ -63,14 +63,14 @@ func TestIntegrationSendingToExternalAlertmanager(t *testing.T) {
}),
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
&fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}, fake_secrets.NewFakeSecretsService())
&fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
}, nil)
// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
// when the first alert triggers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
@ -93,7 +93,7 @@ func TestIntegrationSendingToExternalAlertmanager(t *testing.T) {
// Now, let's remove the Alertmanager from the admin configuration.
mockedGetAdminConfigurations.Return(nil, nil)
// Again, make sure we sync and verify the externalAlertmanagers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
@ -134,7 +134,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
}
fakeDs := &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds1}}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{}, 10*time.Minute,
fakeDs, fake_secrets.NewFakeSecretsService())
fakeDs, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey1.OrgID, SendAlertsTo: models.AllAlertmanagers},
@ -142,7 +142,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
// when the first alert triggers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
@ -167,7 +167,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
}, nil)
// If we sync again, new externalAlertmanagers must have spawned.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 2, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 2, len(alertsRouter.externalAlertmanagersCfgHash))
@ -216,7 +216,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
currentHash := alertsRouter.externalAlertmanagersCfgHash[ruleKey2.OrgID]
// Now, sync again.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
// The hash for org two should not be the same and we should still have two externalAlertmanagers.
require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey2.OrgID], currentHash)
@ -236,7 +236,7 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
currentHash = alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID]
// Now, sync again.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
// The old configuration should not be running.
require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID], currentHash)
@ -249,13 +249,13 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
{OrgID: ruleKey2.OrgID},
}, nil)
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.NotEqual(t, alertsRouter.externalAlertmanagersCfgHash[ruleKey1.OrgID], currentHash)
// Finally, remove everything.
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{}, nil)
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
@ -293,14 +293,14 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
}),
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{},
10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}}, fake_secrets.NewFakeSecretsService())
10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}}, fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures())
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
}, nil)
// Make sure we sync the configuration at least once before the evaluation happens to guarantee the sender is running
// when the first alert triggers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
require.Equal(t, models.AllAlertmanagers, alertsRouter.sendAlertsTo[ruleKey.OrgID])
@ -325,7 +325,7 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
{OrgID: ruleKey.OrgID, SendAlertsTo: models.ExternalAlertmanagers},
}, nil)
// Again, make sure we sync and verify the externalAlertmanagers.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
@ -339,7 +339,7 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
// Again, make sure we sync and verify the externalAlertmanagers.
// externalAlertmanagers should be running even though alerts are being handled externally.
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase())
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 1, len(alertsRouter.externalAlertmanagersCfgHash))
@ -356,6 +356,86 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
require.Len(t, actualAlerts, len(expected))
}
func TestAlertmanagersChoiceWithDisableExternalFeatureToggle(t *testing.T) {
ruleKey := models.GenerateRuleKey(1)
fakeAM := NewFakeExternalAlertmanager(t)
defer fakeAM.Close()
fakeAdminConfigStore := &store.AdminConfigurationStoreMock{}
mockedGetAdminConfigurations := fakeAdminConfigStore.EXPECT().GetAdminConfigurations()
mockedClock := clock.NewMock()
mockedClock.Set(time.Now())
moa := createMultiOrgAlertmanager(t, []int64{1})
appUrl := &url.URL{
Scheme: "http",
Host: "localhost",
}
ds := datasources.DataSource{
URL: fakeAM.Server.URL,
OrgID: ruleKey.OrgID,
Type: datasources.DS_ALERTMANAGER,
JsonData: simplejson.NewFromAny(map[string]any{
"handleGrafanaManagedAlerts": true,
"implementation": "prometheus",
}),
}
var expected []*models2.PostableAlert
alerts := definitions.PostableAlerts{}
for i := 0; i < rand.Intn(5)+1; i++ {
alert := generatePostableAlert(t, mockedClock)
expected = append(expected, &alert)
alerts.PostableAlerts = append(alerts.PostableAlerts, alert)
}
alertsRouter := NewAlertsRouter(moa, fakeAdminConfigStore, mockedClock, appUrl, map[int64]struct{}{},
10*time.Minute, &fake_ds.FakeDataSourceService{DataSources: []*datasources.DataSource{&ds}},
fake_secrets.NewFakeSecretsService(), featuremgmt.WithFeatures(featuremgmt.FlagAlertingDisableSendAlertsExternal))
// Test that we only send to the internal Alertmanager even though the configuration specifies AllAlertmanagers.
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, SendAlertsTo: models.AllAlertmanagers},
}, nil)
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
require.Equal(t, models.InternalAlertmanager, alertsRouter.sendAlertsTo[ruleKey.OrgID])
alertsRouter.Send(context.Background(), ruleKey, alerts)
am, err := moa.AlertmanagerFor(ruleKey.OrgID)
require.NoError(t, err)
actualAlerts, err := am.GetAlerts(context.Background(), true, true, true, nil, "")
require.NoError(t, err)
require.Len(t, actualAlerts, len(expected))
// Test that we still only send to the internal alertmanager even though the configuration specifies ExternalAlertmanagers.
mockedGetAdminConfigurations.Return([]*models.AdminConfiguration{
{OrgID: ruleKey.OrgID, SendAlertsTo: models.ExternalAlertmanagers},
}, nil)
require.NoError(t, alertsRouter.SyncAndApplyConfigFromDatabase(context.Background()))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagers))
require.Equal(t, 0, len(alertsRouter.externalAlertmanagersCfgHash))
require.Equal(t, models.InternalAlertmanager, alertsRouter.sendAlertsTo[ruleKey.OrgID])
alertsRouter.Send(context.Background(), ruleKey, alerts)
am, err = moa.AlertmanagerFor(ruleKey.OrgID)
require.NoError(t, err)
actualAlerts, err = am.GetAlerts(context.Background(), true, true, true, nil, "")
require.NoError(t, err)
require.Len(t, actualAlerts, len(expected))
}
func assertAlertmanagersStatusForOrg(t *testing.T, alertsRouter *AlertsRouter, orgID int64, active, dropped int) {
t.Helper()
require.Eventuallyf(t, func() bool {