Alerting: Support for simplified notification settings in rule API (#81011)

* Add notification settings to storage\domain and API models. Settings are a slice to workaround XORM mapping
* Support validation of notification settings when rules are updated

* Implement route generator for Alertmanager configuration. That fetches all notification settings.
* Update multi-tenant Alertmanager to run the generator before applying the configuration.

* Add notification settings labels to state calculation
* update the Multi-tenant Alertmanager to provide validation for notification settings

* update GET API so only admins can see auto-gen

pkg/tests/api/alerting/api_ruler_test.go

@@ -9,6 +9,7 @@ import (
 	"math/rand"
 	"net/http"
 	"path"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -16,6 +17,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/uuid"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/prometheus/alertmanager/pkg/labels"
 	"github.com/prometheus/common/model"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1816,3 +1818,247 @@ func TestIntegrationHysteresisRule(t *testing.T) {
 	require.NoErrorf(t, json.Unmarshal([]byte(f.At(normalIdx).(string)), &d), body)
 	assert.EqualValuesf(t, 1, d.Values["B"], body)
 }
+
+func TestIntegrationRuleNotificationSettings(t *testing.T) {
+	testinfra.SQLiteIntegrationTest(t)
+
+	// Setup Grafana and its Database. Scheduler is set to evaluate every 1 second
+	dir, p := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+		DisableLegacyAlerting:        true,
+		EnableUnifiedAlerting:        true,
+		DisableAnonymous:             true,
+		AppModeProduction:            true,
+		NGAlertSchedulerBaseInterval: 1 * time.Second,
+		EnableFeatureToggles:         []string{featuremgmt.FlagConfigurableSchedulerTick, featuremgmt.FlagAlertingSimplifiedRouting},
+	})
+
+	grafanaListedAddr, store := testinfra.StartGrafana(t, dir, p)
+
+	// Create a user to make authenticated requests
+	createUser(t, store, user.CreateUserCommand{
+		DefaultOrgRole: string(org.RoleAdmin),
+		Password:       "password",
+		Login:          "grafana",
+	})
+
+	apiClient := newAlertingApiClient(grafanaListedAddr, "grafana", "password")
+
+	folder := "Test-Alerting"
+	apiClient.CreateFolder(t, folder, folder)
+
+	testDataRaw, err := testData.ReadFile(path.Join("test-data", "rule-notification-settings-1-post.json"))
+	require.NoError(t, err)
+
+	type testData struct {
+		RuleGroup    apimodels.PostableRuleGroupConfig
+		Receiver     apimodels.EmbeddedContactPoint
+		TimeInterval apimodels.MuteTimeInterval
+	}
+	var d testData
+	err = json.Unmarshal(testDataRaw, &d)
+	require.NoError(t, err)
+
+	apiClient.EnsureReceiver(t, d.Receiver)
+	apiClient.EnsureMuteTiming(t, d.TimeInterval)
+
+	t.Run("create should fail if receiver does not exist", func(t *testing.T) {
+		var copyD testData
+		err = json.Unmarshal(testDataRaw, &copyD)
+		group := copyD.RuleGroup
+		ns := group.Rules[0].GrafanaManagedAlert.NotificationSettings
+		ns.Receiver = "random-receiver"
+
+		_, status, body := apiClient.PostRulesGroupWithStatus(t, folder, &group)
+		require.Equalf(t, http.StatusBadRequest, status, body)
+		t.Log(body)
+	})
+
+	t.Run("create should fail if mute timing does not exist", func(t *testing.T) {
+		var copyD testData
+		err = json.Unmarshal(testDataRaw, &copyD)
+		group := copyD.RuleGroup
+		ns := group.Rules[0].GrafanaManagedAlert.NotificationSettings
+		ns.MuteTimeIntervals = []string{"random-time-interval"}
+
+		_, status, body := apiClient.PostRulesGroupWithStatus(t, folder, &group)
+		require.Equalf(t, http.StatusBadRequest, status, body)
+		t.Log(body)
+	})
+
+	t.Run("create should fail if group_by does not contain special labels", func(t *testing.T) {
+		var copyD testData
+		err = json.Unmarshal(testDataRaw, &copyD)
+		group := copyD.RuleGroup
+		ns := group.Rules[0].GrafanaManagedAlert.NotificationSettings
+		ns.GroupBy = []string{"label1"}
+
+		_, status, body := apiClient.PostRulesGroupWithStatus(t, folder, &group)
+		require.Equalf(t, http.StatusBadRequest, status, body)
+		t.Log(body)
+	})
+
+	t.Run("should create rule and generate route", func(t *testing.T) {
+		_, status, body := apiClient.PostRulesGroupWithStatus(t, folder, &d.RuleGroup)
+		require.Equalf(t, http.StatusAccepted, status, body)
+		notificationSettings := d.RuleGroup.Rules[0].GrafanaManagedAlert.NotificationSettings
+
+		var routeBody string
+		if !assert.EventuallyWithT(t, func(c *assert.CollectT) {
+			amConfig, status, body := apiClient.GetAlertmanagerConfigWithStatus(t)
+			routeBody = body
+			if !assert.Equalf(t, http.StatusOK, status, body) {
+				return
+			}
+			route := amConfig.AlertmanagerConfig.Route
+
+			if !assert.Len(c, route.Routes, 1) {
+				return
+			}
+
+			// Check that we are in the auto-generated root
+			autogenRoute := route.Routes[0]
+			if !assert.Len(c, autogenRoute.ObjectMatchers, 1) {
+				return
+			}
+			canContinue := assert.Equal(c, ngmodels.AutogeneratedRouteLabel, autogenRoute.ObjectMatchers[0].Name)
+			assert.Equal(c, labels.MatchEqual, autogenRoute.ObjectMatchers[0].Type)
+			assert.Equal(c, "true", autogenRoute.ObjectMatchers[0].Value)
+
+			assert.Equalf(c, route.Receiver, autogenRoute.Receiver, "Autogenerated root receiver must be the default one")
+			assert.Nil(c, autogenRoute.GroupWait)
+			assert.Nil(c, autogenRoute.GroupInterval)
+			assert.Nil(c, autogenRoute.RepeatInterval)
+			assert.Empty(c, autogenRoute.MuteTimeIntervals)
+			assert.Empty(c, autogenRoute.GroupBy)
+			if !canContinue {
+				return
+			}
+			// Now check that the second level is route for receivers
+			if !assert.NotEmpty(c, autogenRoute.Routes) {
+				return
+			}
+			// There can be many routes, for all receivers
+			idx := slices.IndexFunc(autogenRoute.Routes, func(route *apimodels.Route) bool {
+				return route.Receiver == notificationSettings.Receiver
+			})
+			if !assert.GreaterOrEqual(t, idx, 0) {
+				return
+			}
+			receiverRoute := autogenRoute.Routes[idx]
+			if !assert.Len(c, receiverRoute.ObjectMatchers, 1) {
+				return
+			}
+			canContinue = assert.Equal(c, ngmodels.AutogeneratedRouteReceiverNameLabel, receiverRoute.ObjectMatchers[0].Name)
+			assert.Equal(c, labels.MatchEqual, receiverRoute.ObjectMatchers[0].Type)
+			assert.Equal(c, notificationSettings.Receiver, receiverRoute.ObjectMatchers[0].Value)
+
+			assert.Equal(c, notificationSettings.Receiver, receiverRoute.Receiver)
+			assert.Nil(c, receiverRoute.GroupWait)
+			assert.Nil(c, receiverRoute.GroupInterval)
+			assert.Nil(c, receiverRoute.RepeatInterval)
+			assert.Empty(c, receiverRoute.MuteTimeIntervals)
+			var groupBy []string
+			for _, name := range receiverRoute.GroupBy {
+				groupBy = append(groupBy, string(name))
+			}
+			slices.Sort(groupBy)
+			assert.EqualValues(c, []string{"alertname", "grafana_folder"}, groupBy)
+			if !canContinue {
+				return
+			}
+			// Now check that we created the 3rd level for specific combination of settings
+			if !assert.Lenf(c, receiverRoute.Routes, 1, "Receiver route should contain one options route") {
+				return
+			}
+			optionsRoute := receiverRoute.Routes[0]
+			if !assert.Len(c, optionsRoute.ObjectMatchers, 1) {
+				return
+			}
+			assert.Equal(c, ngmodels.AutogeneratedRouteSettingsHashLabel, optionsRoute.ObjectMatchers[0].Name)
+			assert.Equal(c, labels.MatchEqual, optionsRoute.ObjectMatchers[0].Type)
+			assert.EqualValues(c, notificationSettings.GroupWait, optionsRoute.GroupWait)
+			assert.EqualValues(c, notificationSettings.GroupInterval, optionsRoute.GroupInterval)
+			assert.EqualValues(c, notificationSettings.RepeatInterval, optionsRoute.RepeatInterval)
+			assert.EqualValues(c, notificationSettings.MuteTimeIntervals, optionsRoute.MuteTimeIntervals)
+			groupBy = nil
+			for _, name := range optionsRoute.GroupBy {
+				groupBy = append(groupBy, string(name))
+			}
+			assert.EqualValues(c, notificationSettings.GroupBy, groupBy)
+		}, 10*time.Second, 1*time.Second) {
+			t.Logf("config: %s", routeBody)
+		}
+	})
+
+	t.Run("should correctly create alerts", func(t *testing.T) {
+		var response string
+		if !assert.EventuallyWithT(t, func(c *assert.CollectT) {
+			groups, status, body := apiClient.GetActiveAlertsWithStatus(t)
+			require.Equalf(t, http.StatusOK, status, body)
+			response = body
+			if len(groups) == 0 {
+				return
+			}
+			g := groups[0]
+			alert := g.Alerts[0]
+			assert.Contains(c, alert.Labels, ngmodels.AutogeneratedRouteLabel)
+			assert.Equal(c, "true", alert.Labels[ngmodels.AutogeneratedRouteLabel])
+			assert.Contains(c, alert.Labels, ngmodels.AutogeneratedRouteReceiverNameLabel)
+			assert.Equal(c, d.Receiver.Name, alert.Labels[ngmodels.AutogeneratedRouteReceiverNameLabel])
+			assert.Contains(c, alert.Labels, ngmodels.AutogeneratedRouteSettingsHashLabel)
+			assert.NotEmpty(c, alert.Labels[ngmodels.AutogeneratedRouteSettingsHashLabel])
+		}, 10*time.Second, 1*time.Second) {
+			t.Logf("response: %s", response)
+		}
+	})
+
+	t.Run("should update rule with empty settings and delete route", func(t *testing.T) {
+		var copyD testData
+		err = json.Unmarshal(testDataRaw, &copyD)
+		group := copyD.RuleGroup
+		notificationSettings := group.Rules[0].GrafanaManagedAlert.NotificationSettings
+		group.Rules[0].GrafanaManagedAlert.NotificationSettings = nil
+
+		_, status, body := apiClient.PostRulesGroupWithStatus(t, folder, &group)
+		require.Equalf(t, http.StatusAccepted, status, body)
+
+		var routeBody string
+		if !assert.EventuallyWithT(t, func(c *assert.CollectT) {
+			amConfig, status, body := apiClient.GetAlertmanagerConfigWithStatus(t)
+			routeBody = body
+			if !assert.Equalf(t, http.StatusOK, status, body) {
+				return
+			}
+			route := amConfig.AlertmanagerConfig.Route
+
+			if !assert.Len(c, route.Routes, 1) {
+				return
+			}
+			// Check that we are in the auto-generated root
+			autogenRoute := route.Routes[0]
+			if !assert.Len(c, autogenRoute.ObjectMatchers, 1) {
+				return
+			}
+			if !assert.Equal(c, ngmodels.AutogeneratedRouteLabel, autogenRoute.ObjectMatchers[0].Name) {
+				return
+			}
+			// Now check that the second level is route for receivers
+			if !assert.NotEmpty(c, autogenRoute.Routes) {
+				return
+			}
+			// There can be many routes, for all receivers
+			idx := slices.IndexFunc(autogenRoute.Routes, func(route *apimodels.Route) bool {
+				return route.Receiver == notificationSettings.Receiver
+			})
+			if !assert.GreaterOrEqual(t, idx, 0) {
+				return
+			}
+			receiverRoute := autogenRoute.Routes[idx]
+			if !assert.Empty(c, receiverRoute.Routes) {
+				return
+			}
+		}, 10*time.Second, 1*time.Second) {
+			t.Logf("config: %s", routeBody)
+		}
+	})
+}

pkg/tests/api/alerting/test-data/rule-notification-settings-1-post.json

@@ -0,0 +1,58 @@
+{
+  "ruleGroup" : {
+    "name": "Group1",
+    "interval": "1m",
+    "rules": [
+      {
+        "for": "0",
+        "labels": {
+          "label1": "test-label"
+        },
+        "annotations": {
+          "annotation": "test-annotation"
+        },
+        "grafana_alert": {
+          "title": "Rule1",
+          "condition": "A",
+          "data": [
+            {
+              "refId": "A",
+              "datasourceUid": "__expr__",
+              "model": {
+                "expression": "0 > 0",
+                "type": "math"
+              }
+            }
+          ],
+          "no_data_state": "NoData",
+          "exec_err_state": "Alerting",
+          "notification_settings": {
+            "receiver": "rule-receiver",
+            "group_by": [
+              "alertname",
+              "grafana_folder",
+              "label1"
+            ],
+            "group_wait": "100ms",
+            "group_interval": "5s",
+            "repeat_interval": "1d",
+            "mute_time_intervals": [
+              "rule-time-interval"
+            ]
+          }
+        }
+      }
+    ]
+  },
+  "receiver": {
+    "name": "rule-receiver",
+    "type": "webhook",
+    "settings": {
+      "url": "http://localhost:3000/_callback"
+    }
+  },
+  "timeInterval": {
+    "name": "rule-time-interval",
+    "time_intervals":[{"times":[{"start_time":"10:00","end_time":"12:00"}]}]
+  }
+}

pkg/tests/api/alerting/testing.go

@@ -226,13 +226,14 @@ func convertGettableGrafanaRuleToPostable(gettable *apimodels.GettableGrafanaRul
 		return nil
 	}
 	return &apimodels.PostableGrafanaRule{
-		Title:        gettable.Title,
-		Condition:    gettable.Condition,
-		Data:         gettable.Data,
-		UID:          gettable.UID,
-		NoDataState:  gettable.NoDataState,
-		ExecErrState: gettable.ExecErrState,
-		IsPaused:     &gettable.IsPaused,
+		Title:                gettable.Title,
+		Condition:            gettable.Condition,
+		Data:                 gettable.Data,
+		UID:                  gettable.UID,
+		NoDataState:          gettable.NoDataState,
+		ExecErrState:         gettable.ExecErrState,
+		IsPaused:             &gettable.IsPaused,
+		NotificationSettings: gettable.NotificationSettings,
 	}
 }
 
@@ -711,6 +712,13 @@ func (a apiClient) CreateMuteTimingWithStatus(t *testing.T, interval apimodels.M
 	return sendRequest[apimodels.MuteTimeInterval](t, req, http.StatusCreated)
 }
 
+func (a apiClient) EnsureMuteTiming(t *testing.T, interval apimodels.MuteTimeInterval) {
+	t.Helper()
+
+	_, status, body := a.CreateMuteTimingWithStatus(t, interval)
+	require.Equalf(t, http.StatusCreated, status, body)
+}
+
 func (a apiClient) UpdateMuteTimingWithStatus(t *testing.T, interval apimodels.MuteTimeInterval) (apimodels.MuteTimeInterval, int, string) {
 	t.Helper()
 
@@ -810,6 +818,43 @@ func (a apiClient) GetTimeIntervalByNameWithStatus(t *testing.T, name string) (a
 	return sendRequest[apimodels.GettableTimeIntervals](t, req, http.StatusOK)
 }
 
+func (a apiClient) CreateReceiverWithStatus(t *testing.T, receiver apimodels.EmbeddedContactPoint) (apimodels.EmbeddedContactPoint, int, string) {
+	t.Helper()
+
+	buf := bytes.Buffer{}
+	enc := json.NewEncoder(&buf)
+	err := enc.Encode(receiver)
+	require.NoError(t, err)
+
+	req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s/api/v1/provisioning/contact-points", a.url), &buf)
+	req.Header.Add("Content-Type", "application/json")
+	require.NoError(t, err)
+
+	return sendRequest[apimodels.EmbeddedContactPoint](t, req, http.StatusAccepted)
+}
+
+func (a apiClient) EnsureReceiver(t *testing.T, receiver apimodels.EmbeddedContactPoint) {
+	t.Helper()
+
+	_, status, body := a.CreateReceiverWithStatus(t, receiver)
+	require.Equalf(t, http.StatusAccepted, status, body)
+}
+
+func (a apiClient) GetAlertmanagerConfigWithStatus(t *testing.T) (apimodels.GettableUserConfig, int, string) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/alertmanager/grafana/config/api/v1/alerts", a.url), nil)
+	require.NoError(t, err)
+
+	return sendRequest[apimodels.GettableUserConfig](t, req, http.StatusOK)
+}
+
+func (a apiClient) GetActiveAlertsWithStatus(t *testing.T) (apimodels.AlertGroups, int, string) {
+	t.Helper()
+	req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/api/alertmanager/grafana/api/v2/alerts/groups", a.url), nil)
+	require.NoError(t, err)
+	return sendRequest[apimodels.AlertGroups](t, req, http.StatusOK)
+}
+
 func sendRequest[T any](t *testing.T, req *http.Request, successStatusCode int) (T, int, string) {
 	client := &http.Client{}
 	resp, err := client.Do(req)