Alerting: Remove ngalert feature toggle and introduce two new settings for enabling Grafana 8 alerts and disabling them for specific organisations (#38746)

* Remove `ngalert` feature toggle

* Update frontend

Remove all references of ngalert feature toggle

* Update docs

* Disable unified alerting for specific orgs

* Add backend tests

* Apply suggestions from code review

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Disabled unified alerting by default

* Ensure backward compatibility with old ngalert feature toggle

* Apply suggestions from code review

Co-authored-by: gotjosh <josue@grafana.com>

pkg/tests/api/alerting/api_admin_configuration_test.go

@@ -20,10 +20,13 @@ import (
 )
 
 func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
+	const disableOrgID int64 = 3
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles:           []string{"ngalert"},
+		DisableLegacyAlerting:          true,
+		EnableUnifiedAlerting:          true,
 		DisableAnonymous:               true,
 		NGAlertAdminConfigPollInterval: 2 * time.Second,
+		UnifiedAlertingDisabledOrgs:    []int64{disableOrgID}, // disable unified alerting for organisation 3
 	})
 
 	grafanaListedAddr, s := testinfra.StartGrafana(t, dir, path)
@@ -31,15 +34,29 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 	s.Bus = bus.GetBus()
 
 	// Create a user to make authenticated requests
-	createUser(t, s, models.CreateUserCommand{
+	userID := createUser(t, s, models.CreateUserCommand{
 		DefaultOrgRole: string(models.ROLE_ADMIN),
 		Login:          "grafana",
 		Password:       "password",
 	})
 
+	// create another organisation
+	orgID := createOrg(t, s, "another org", userID)
+	// ensure that the orgID is 3 (the disabled org)
+	require.Equal(t, disableOrgID, orgID)
+
+	// create user under different organisation
+	createUser(t, s, models.CreateUserCommand{
+		DefaultOrgRole: string(models.ROLE_ADMIN),
+		Password:       "admin-42",
+		Login:          "admin-42",
+		OrgId:          orgID,
+	})
+
 	// Create a couple of "fake" Alertmanagers
 	fakeAM1 := schedule.NewFakeExternalAlertmanager(t)
 	fakeAM2 := schedule.NewFakeExternalAlertmanager(t)
+	fakeAM3 := schedule.NewFakeExternalAlertmanager(t)
 
 	// Now, let's test the configuration API.
 	{
@@ -50,7 +67,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		require.JSONEq(t, string(b), "{\"message\": \"no admin configuration available\"}")
 	}
 
-	// Now, lets re-set external Alertmanagers.
+	// Now, lets re-set external Alertmanagers for main organisation.
 	{
 		ac := apimodels.PostableNGalertConfig{
 			Alertmanagers: []string{fakeAM1.URL(), fakeAM2.URL()},
@@ -76,7 +93,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 		require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\",\"%s\"]}\n", fakeAM1.URL(), fakeAM2.URL()))
 	}
 
-	// With the configuration set, we should eventually discover those Alertmanagers set.
+	// With the configuration set, we should eventually discover those Alertmanagers.
 	{
 		alertsURL := fmt.Sprintf("http://grafana:password@%s/api/v1/ngalert/alertmanagers", grafanaListedAddr)
 		require.Eventually(t, func() bool {
@@ -88,7 +105,7 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 			require.NoError(t, json.Unmarshal(b, &alertmanagers))
 
 			return len(alertmanagers.Data.Active) == 2
-		}, 80*time.Second, 4*time.Second)
+		}, 16*time.Second, 8*time.Second) // the sync interval is 2s so after 8s all alertmanagers most probably are started
 	}
 
 	// Now, let's set an alert that should fire as quickly as possible.
@@ -148,4 +165,45 @@ func TestAdminConfiguration_SendingToExternalAlertmanagers(t *testing.T) {
 			return fakeAM1.AlertsCount() == 1 && fakeAM2.AlertsCount() == 1
 		}, 60*time.Second, 5*time.Second)
 	}
+
+	// Now, lets re-set external Alertmanagers for the other organisation.
+	{
+		ac := apimodels.PostableNGalertConfig{
+			Alertmanagers: []string{fakeAM3.URL()},
+		}
+		buf := bytes.Buffer{}
+		enc := json.NewEncoder(&buf)
+		err := enc.Encode(&ac)
+		require.NoError(t, err)
+
+		alertsURL := fmt.Sprintf("http://admin-42:admin-42@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
+		resp := postRequest(t, alertsURL, buf.String(), http.StatusCreated) // nolint
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.JSONEq(t, string(b), "{\"message\": \"admin configuration updated\"}")
+	}
+
+	// If we get the configuration again, it shows us what we've set.
+	{
+		alertsURL := fmt.Sprintf("http://admin-42:admin-42@%s/api/v1/ngalert/admin_config", grafanaListedAddr)
+		resp := getRequest(t, alertsURL, http.StatusOK) // nolint
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.JSONEq(t, string(b), fmt.Sprintf("{\"alertmanagers\":[\"%s\"]}\n", fakeAM3.URL()))
+	}
+
+	// With the configuration set, we should eventually not discover Alertmanagers.
+	{
+		alertsURL := fmt.Sprintf("http://admin-42:admin-42@%s/api/v1/ngalert/alertmanagers", grafanaListedAddr)
+		require.Eventually(t, func() bool {
+			resp := getRequest(t, alertsURL, http.StatusOK) // nolint
+			b, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+
+			var alertmanagers apimodels.GettableAlertmanagers
+			require.NoError(t, json.Unmarshal(b, &alertmanagers))
+
+			return len(alertmanagers.Data.Active) == 0
+		}, 16*time.Second, 8*time.Second) // the sync interval is 2s so after 8s all alertmanagers (if any) most probably are started
+	}
 }

pkg/tests/api/alerting/api_alertmanager_configuration_test.go

@@ -18,7 +18,8 @@ import (
 
 func TestAlertmanagerConfigurationIsTransactional(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles:                  []string{"ngalert"},
+		DisableLegacyAlerting:                 true,
+		EnableUnifiedAlerting:                 true,
 		NGAlertAlertmanagerConfigPollInterval: 2 * time.Second,
 		DisableAnonymous:                      true,
 	})
@@ -127,8 +128,9 @@ func TestAlertmanagerConfigurationIsTransactional(t *testing.T) {
 
 func TestAlertmanagerConfigurationPersistSecrets(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)

pkg/tests/api/alerting/api_alertmanager_test.go

@@ -29,8 +29,9 @@ import (
 
 func TestAMConfigAccess(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -387,8 +388,9 @@ func TestAMConfigAccess(t *testing.T) {
 
 func TestAlertAndGroupsQuery(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -552,10 +554,11 @@ func TestAlertAndGroupsQuery(t *testing.T) {
 func TestRulerAccess(t *testing.T) {
 	// Setup Grafana and its Database
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		EnableQuota:          true,
-		DisableAnonymous:     true,
-		ViewersCanEdit:       true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		EnableQuota:           true,
+		DisableAnonymous:      true,
+		ViewersCanEdit:        true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -678,10 +681,11 @@ func TestRulerAccess(t *testing.T) {
 func TestDeleteFolderWithRules(t *testing.T) {
 	// Setup Grafana and its Database
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		EnableQuota:          true,
-		DisableAnonymous:     true,
-		ViewersCanEdit:       true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		EnableQuota:           true,
+		DisableAnonymous:      true,
+		ViewersCanEdit:        true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -837,9 +841,10 @@ func TestDeleteFolderWithRules(t *testing.T) {
 func TestAlertRuleCRUD(t *testing.T) {
 	// Setup Grafana and its Database
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		EnableQuota:          true,
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		EnableQuota:           true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -1905,7 +1910,8 @@ func TestAlertRuleCRUD(t *testing.T) {
 func TestAlertmanagerStatus(t *testing.T) {
 	// Setup Grafana and its Database
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
 	})
 
 	grafanaListedAddr, _ := testinfra.StartGrafana(t, dir, path)
@@ -1965,9 +1971,10 @@ func TestAlertmanagerStatus(t *testing.T) {
 func TestQuota(t *testing.T) {
 	// Setup Grafana and its Database
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		EnableQuota:          true,
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		EnableQuota:           true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -2207,9 +2214,10 @@ func TestQuota(t *testing.T) {
 func TestEval(t *testing.T) {
 	// Setup Grafana and its Database
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		EnableQuota:          true,
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		EnableQuota:           true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)

pkg/tests/api/alerting/api_available_channel_test.go

@@ -15,8 +15,9 @@ import (
 
 func TestAvailableChannels(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)

pkg/tests/api/alerting/api_notification_channel_test.go

@@ -34,7 +34,8 @@ func TestTestReceivers(t *testing.T) {
 	t.Run("assert no receivers returns 400 Bad Request", func(t *testing.T) {
 		// Setup Grafana and its Database
 		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-			EnableFeatureToggles: []string{"ngalert"},
+			DisableLegacyAlerting: true,
+			EnableUnifiedAlerting: true,
 		})
 
 		grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -64,7 +65,8 @@ func TestTestReceivers(t *testing.T) {
 	t.Run("assert working receiver returns OK", func(t *testing.T) {
 		// Setup Grafana and its Database
 		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-			EnableFeatureToggles: []string{"ngalert"},
+			DisableLegacyAlerting: true,
+			EnableUnifiedAlerting: true,
 		})
 
 		grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -131,7 +133,8 @@ func TestTestReceivers(t *testing.T) {
 	t.Run("assert invalid receiver returns 400 Bad Request", func(t *testing.T) {
 		// Setup Grafana and its Database
 		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-			EnableFeatureToggles: []string{"ngalert"},
+			DisableLegacyAlerting: true,
+			EnableUnifiedAlerting: true,
 		})
 
 		grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -194,7 +197,8 @@ func TestTestReceivers(t *testing.T) {
 	t.Run("assert timed out receiver returns 408 Request Timeout", func(t *testing.T) {
 		// Setup Grafana and its Database
 		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-			EnableFeatureToggles: []string{"ngalert"},
+			DisableLegacyAlerting: true,
+			EnableUnifiedAlerting: true,
 		})
 
 		grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -266,7 +270,8 @@ func TestTestReceivers(t *testing.T) {
 	t.Run("assert multiple different errors returns 207 Multi Status", func(t *testing.T) {
 		// Setup Grafana and its Database
 		dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-			EnableFeatureToggles: []string{"ngalert"},
+			DisableLegacyAlerting: true,
+			EnableUnifiedAlerting: true,
 		})
 
 		grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -359,8 +364,9 @@ func TestTestReceivers(t *testing.T) {
 
 func TestNotificationChannels(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, s := testinfra.StartGrafana(t, dir, path)

pkg/tests/api/alerting/api_prometheus_test.go

@@ -21,8 +21,9 @@ import (
 
 func TestPrometheusRules(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -265,8 +266,9 @@ func TestPrometheusRules(t *testing.T) {
 
 func TestPrometheusRulesPermissions(t *testing.T) {
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)

pkg/tests/api/alerting/api_ruler_test.go

@@ -22,8 +22,9 @@ import (
 func TestAlertRulePermissions(t *testing.T) {
 	// Setup Grafana and its Database
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		DisableAnonymous:     true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		DisableAnonymous:      true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)
@@ -306,10 +307,11 @@ func createRule(t *testing.T, grafanaListedAddr string, folder string, user, pas
 func TestAlertRuleConflictingTitle(t *testing.T) {
 	// Setup Grafana and its Database
 	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
-		EnableFeatureToggles: []string{"ngalert"},
-		EnableQuota:          true,
-		DisableAnonymous:     true,
-		ViewersCanEdit:       true,
+		DisableLegacyAlerting: true,
+		EnableUnifiedAlerting: true,
+		EnableQuota:           true,
+		DisableAnonymous:      true,
+		ViewersCanEdit:        true,
 	})
 
 	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, path)

pkg/tests/testinfra/testinfra.go

@@ -194,6 +194,14 @@ func CreateGrafDir(t *testing.T, opts ...GrafanaOpts) (string, string) {
 	_, err = alertingSect.NewKey("max_attempts", "3")
 	require.NoError(t, err)
 
+	getOrCreateSection := func(name string) (*ini.Section, error) {
+		section, err := cfg.GetSection(name)
+		if err != nil {
+			return cfg.NewSection(name)
+		}
+		return section, err
+	}
+
 	for _, o := range opts {
 		if o.EnableCSP {
 			securitySect, err := cfg.NewSection("security")
@@ -214,7 +222,7 @@ func CreateGrafDir(t *testing.T, opts ...GrafanaOpts) (string, string) {
 			require.NoError(t, err)
 		}
 		if o.NGAlertAlertmanagerConfigPollInterval != 0 {
-			ngalertingSection, err := cfg.NewSection("unified_alerting")
+			ngalertingSection, err := getOrCreateSection("unified_alerting")
 			require.NoError(t, err)
 			_, err = ngalertingSection.NewKey("alertmanager_config_poll_interval", o.NGAlertAlertmanagerConfigPollInterval.String())
 			require.NoError(t, err)
@@ -247,6 +255,25 @@ func CreateGrafDir(t *testing.T, opts ...GrafanaOpts) (string, string) {
 			_, err = usersSection.NewKey("viewers_can_edit", "true")
 			require.NoError(t, err)
 		}
+		if o.DisableLegacyAlerting {
+			alertingSection, err := cfg.GetSection("alerting")
+			require.NoError(t, err)
+			_, err = alertingSection.NewKey("enabled", "false")
+			require.NoError(t, err)
+		}
+		if o.EnableUnifiedAlerting {
+			unifiedAlertingSection, err := getOrCreateSection("unified_alerting")
+			require.NoError(t, err)
+			_, err = unifiedAlertingSection.NewKey("enabled", "true")
+			require.NoError(t, err)
+		}
+		if len(o.UnifiedAlertingDisabledOrgs) > 0 {
+			unifiedAlertingSection, err := getOrCreateSection("unified_alerting")
+			require.NoError(t, err)
+			disableOrgStr := strings.Join(strings.Split(strings.Trim(fmt.Sprint(o.UnifiedAlertingDisabledOrgs), "[]"), " "), ",")
+			_, err = unifiedAlertingSection.NewKey("disabled_orgs", disableOrgStr)
+			require.NoError(t, err)
+		}
 	}
 
 	cfgPath := filepath.Join(cfgDir, "test.ini")
@@ -270,4 +297,7 @@ type GrafanaOpts struct {
 	CatalogAppEnabled                     bool
 	ViewersCanEdit                        bool
 	PluginAdminEnabled                    bool
+	DisableLegacyAlerting                 bool
+	EnableUnifiedAlerting                 bool
+	UnifiedAlertingDisabledOrgs           []int64
 }