From ec3214bac21c10da710018ee0a6021017b180219 Mon Sep 17 00:00:00 2001 From: Ganesh Vernekar <15064823+codesome@users.noreply.github.com> Date: Thu, 13 May 2021 22:58:19 +0530 Subject: [PATCH] NGAlert: Add integration tests for notification channels (#33431) * NGAlert: Add integration tests for notification channels Signed-off-by: Ganesh Vernekar * Fix the failing tests Signed-off-by: Ganesh Vernekar * Fix review comments Signed-off-by: Ganesh Vernekar * Override creation of rule UID, remove only namespace UID Signed-off-by: Ganesh Vernekar --- pkg/services/ngalert/notifier/alertmanager.go | 2 +- .../ngalert/notifier/channels/pagerduty.go | 4 +- .../ngalert/notifier/channels/slack.go | 10 +- .../ngalert/notifier/channels/telegram.go | 6 +- pkg/services/ngalert/store/alert_rule.go | 7 +- .../alerting/api_available_channel_test.go | 9 +- .../alerting/api_notification_channel_test.go | 976 ++++++++++++++++++ pkg/tests/testinfra/testinfra.go | 5 + 8 files changed, 1004 insertions(+), 15 deletions(-) create mode 100644 pkg/tests/api/alerting/api_notification_channel_test.go diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index dc22575e252..77e53d4907b 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -446,7 +446,7 @@ func (am *Alertmanager) PutAlerts(postableAlerts apimodels.PostableAlerts) error UpdatedAt: now, } for k, v := range a.Labels { - if len(v) == 0 { // Skip empty labels. + if len(v) == 0 || k == ngmodels.NamespaceUIDLabel { // Skip empty and namespace UID labels. continue } alert.Alert.Labels[model.LabelName(k)] = model.LabelValue(v) diff --git a/pkg/services/ngalert/notifier/channels/pagerduty.go b/pkg/services/ngalert/notifier/channels/pagerduty.go index 9b1539582a6..a22566f2173 100644 --- a/pkg/services/ngalert/notifier/channels/pagerduty.go +++ b/pkg/services/ngalert/notifier/channels/pagerduty.go @@ -25,7 +25,7 @@ const ( ) var ( - pagerdutyEventAPIURL = "https://events.pagerduty.com/v2/enqueue" + PagerdutyEventAPIURL = "https://events.pagerduty.com/v2/enqueue" ) // PagerdutyNotifier is responsible for sending @@ -93,7 +93,7 @@ func (pn *PagerdutyNotifier) Notify(ctx context.Context, as ...*types.Alert) (bo pn.log.Info("Notifying Pagerduty", "event_type", eventType) cmd := &models.SendWebhookSync{ - Url: pagerdutyEventAPIURL, + Url: PagerdutyEventAPIURL, Body: string(body), HttpMethod: "POST", HttpHeader: map[string]string{ diff --git a/pkg/services/ngalert/notifier/channels/slack.go b/pkg/services/ngalert/notifier/channels/slack.go index b1ffc75fced..08506e7468f 100644 --- a/pkg/services/ngalert/notifier/channels/slack.go +++ b/pkg/services/ngalert/notifier/channels/slack.go @@ -50,7 +50,7 @@ type SlackNotifier struct { var reRecipient *regexp.Regexp = regexp.MustCompile("^((@[a-z0-9][a-zA-Z0-9._-]*)|(#[^ .A-Z]{1,79})|([a-zA-Z0-9]+))$") -const slackAPIEndpoint = "https://slack.com/api/chat.postMessage" +var SlackAPIEndpoint = "https://slack.com/api/chat.postMessage" // NewSlackNotifier is the constructor for the Slack notifier func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*SlackNotifier, error) { @@ -60,7 +60,7 @@ func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*S slackURL := model.DecryptedValue("url", model.Settings.Get("url").MustString()) if slackURL == "" { - slackURL = slackAPIEndpoint + slackURL = SlackAPIEndpoint } apiURL, err := url.Parse(slackURL) if err != nil { @@ -72,7 +72,7 @@ func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*S if !reRecipient.MatchString(recipient) { return nil, alerting.ValidationError{Reason: fmt.Sprintf("recipient on invalid format: %q", recipient)} } - } else if apiURL.String() == slackAPIEndpoint { + } else if apiURL.String() == SlackAPIEndpoint { return nil, alerting.ValidationError{ Reason: "recipient must be specified when using the Slack chat API", } @@ -104,7 +104,7 @@ func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*S } token := model.DecryptedValue("token", model.Settings.Get("token").MustString()) - if token == "" && apiURL.String() == slackAPIEndpoint { + if token == "" && apiURL.String() == SlackAPIEndpoint { return nil, alerting.ValidationError{ Reason: "token must be specified when using the Slack chat API", } @@ -172,7 +172,7 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, request.Header.Set("Content-Type", "application/json") request.Header.Set("User-Agent", "Grafana") if sn.Token == "" { - if sn.URL.String() == slackAPIEndpoint { + if sn.URL.String() == SlackAPIEndpoint { panic("Token should be set when using the Slack chat API") } } else { diff --git a/pkg/services/ngalert/notifier/channels/telegram.go b/pkg/services/ngalert/notifier/channels/telegram.go index 140f02946fd..9d7cbb9daf0 100644 --- a/pkg/services/ngalert/notifier/channels/telegram.go +++ b/pkg/services/ngalert/notifier/channels/telegram.go @@ -18,8 +18,8 @@ import ( old_notifiers "github.com/grafana/grafana/pkg/services/alerting/notifiers" ) -const ( - telegramAPIURL = "https://api.telegram.org/bot%s/sendMessage" +var ( + TelegramAPIURL = "https://api.telegram.org/bot%s/sendMessage" ) // TelegramNotifier is responsible for sending @@ -90,7 +90,7 @@ func (tn *TelegramNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo tn.log.Info("sending telegram notification", "chat_id", tn.ChatID) cmd := &models.SendWebhookSync{ - Url: fmt.Sprintf(telegramAPIURL, tn.BotToken), + Url: fmt.Sprintf(TelegramAPIURL, tn.BotToken), Body: body.String(), HttpMethod: "POST", HttpHeader: map[string]string{ diff --git a/pkg/services/ngalert/store/alert_rule.go b/pkg/services/ngalert/store/alert_rule.go index 9a7e22c3561..ad89a459cd6 100644 --- a/pkg/services/ngalert/store/alert_rule.go +++ b/pkg/services/ngalert/store/alert_rule.go @@ -203,7 +203,7 @@ func (st DBstore) UpsertAlertRules(rules []UpsertRule) error { var parentVersion int64 switch r.Existing { case nil: // new rule - uid, err := generateNewAlertRuleUID(sess, r.New.OrgID) + uid, err := GenerateNewAlertRuleUID(sess, r.New.OrgID, r.New.Title) if err != nil { return fmt.Errorf("failed to generate UID for alert rule %q: %w", r.New.Title, err) } @@ -411,7 +411,10 @@ func (st DBstore) GetAlertRulesForScheduling(query *ngmodels.ListAlertRulesQuery }) } -func generateNewAlertRuleUID(sess *sqlstore.DBSession, orgID int64) (string, error) { +// GenerateNewAlertRuleUID generates a unique UID for a rule. +// This is set as a variable so that the tests can override it. +// The ruleTitle is only used by the mocked functions. +var GenerateNewAlertRuleUID = func(sess *sqlstore.DBSession, orgID int64, ruleTitle string) (string, error) { for i := 0; i < 3; i++ { uid := util.GenerateShortUID() diff --git a/pkg/tests/api/alerting/api_available_channel_test.go b/pkg/tests/api/alerting/api_available_channel_test.go index 82cc4e46c84..bef8289893d 100644 --- a/pkg/tests/api/alerting/api_available_channel_test.go +++ b/pkg/tests/api/alerting/api_available_channel_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/tests/testinfra" ) @@ -15,13 +16,17 @@ import ( func TestAvailableChannels(t *testing.T) { dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ EnableFeatureToggles: []string{"ngalert"}, - AnonymousUserRole: models.ROLE_EDITOR, + DisableAnonymous: true, }) store := testinfra.SetUpDatabase(t, dir) + store.Bus = bus.GetBus() grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store) - alertsURL := fmt.Sprintf("http://%s/api/alert-notifiers", grafanaListedAddr) + // Create a user to make authenticated requests + require.NoError(t, createUser(t, store, models.ROLE_EDITOR, "grafana", "password")) + + alertsURL := fmt.Sprintf("http://grafana:password@%s/api/alert-notifiers", grafanaListedAddr) // nolint:gosec resp, err := http.Get(alertsURL) require.NoError(t, err) diff --git a/pkg/tests/api/alerting/api_notification_channel_test.go b/pkg/tests/api/alerting/api_notification_channel_test.go new file mode 100644 index 00000000000..977fc8e1f9b --- /dev/null +++ b/pkg/tests/api/alerting/api_notification_channel_test.go @@ -0,0 +1,976 @@ +package alerting + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/http" + "regexp" + "strings" + "sync" + "testing" + "time" + + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/models" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/services/ngalert/notifier/channels" + "github.com/grafana/grafana/pkg/services/ngalert/store" + "github.com/grafana/grafana/pkg/services/sqlstore" + "github.com/grafana/grafana/pkg/tests/testinfra" +) + +func TestNotificationChannels(t *testing.T) { + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ + EnableFeatureToggles: []string{"ngalert"}, + DisableAnonymous: true, + }) + + s := testinfra.SetUpDatabase(t, dir) + s.Bus = bus.GetBus() + grafanaListedAddr := testinfra.StartGrafana(t, dir, path, s) + + mockChannel := newMockNotificationChannel(t, grafanaListedAddr) + amConfig := getAlertmanagerConfig(mockChannel.server.Addr) + + // Overriding some URLs to send to the mock channel. + channels.SlackAPIEndpoint = fmt.Sprintf("http://%s/slack_recvX/slack_testX", mockChannel.server.Addr) + channels.PagerdutyEventAPIURL = fmt.Sprintf("http://%s/pagerduty_recvX/pagerduty_testX", mockChannel.server.Addr) + channels.TelegramAPIURL = fmt.Sprintf("http://%s/telegram_recv/bot%%s", mockChannel.server.Addr) + + // Create a user to make authenticated requests + require.NoError(t, createUser(t, s, models.ROLE_EDITOR, "grafana", "password")) + + { + // There are no notification channel config initially. + alertsURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr) + _ = getRequest(t, alertsURL, http.StatusNotFound) // nolint + } + + { + // Create the namespace we'll save our alerts to. + require.NoError(t, createFolder(t, s, 0, "default")) + + // Post the alertmanager config. + u := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr) + postRequest(t, u, amConfig, http.StatusAccepted) + + // Verifying that all the receivers and routes have been registered. + alertsURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr) + resp := getRequest(t, alertsURL, http.StatusOK) // nolint + b := getBody(t, resp.Body) + require.JSONEq(t, getExpAlertmanagerConfigFromAPI(mockChannel.server.Addr), b) + } + + { + // Create rules that will fire as quickly as possible + + originalFunction := store.GenerateNewAlertRuleUID + t.Cleanup(func() { + store.GenerateNewAlertRuleUID = originalFunction + }) + store.GenerateNewAlertRuleUID = func(_ *sqlstore.DBSession, _ int64, ruleTitle string) (string, error) { + return "UID_" + ruleTitle, nil + } + + rulesConfig := getRulesConfig(t) + u := fmt.Sprintf("http://grafana:password@%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr) + postRequest(t, u, rulesConfig, http.StatusAccepted) + } + + // Eventually, we'll get all the desired alerts. + // nolint:gosec + require.Eventually(t, func() bool { + return mockChannel.totalNotifications() == len(alertNames) && + mockChannel.matchesExpNotifications(expNotifications) + }, 25*time.Second, 1*time.Second) + + require.NoError(t, mockChannel.Close()) +} + +func getAlertmanagerConfig(channelAddr string) string { + return strings.ReplaceAll(alertmanagerConfig, "CHANNEL_ADDR", channelAddr) +} + +func getExpAlertmanagerConfigFromAPI(channelAddr string) string { + return strings.ReplaceAll(expAlertmanagerConfigFromAPI, "CHANNEL_ADDR", channelAddr) +} + +// alertNames are name of alerts to be sent. This should be in sync with +// the routes that we define in Alertmanager config. +// EmailAlert and TelegramAlert are missing because they don't +// send a JSON. Email and POST body are yet to be supported in the tests. +var alertNames = []string{"DingDingAlert", "SlackAlert1", "SlackAlert2", "PagerdutyAlert", "TeamsAlert", "WebhookAlert"} + +func getRulesConfig(t *testing.T) string { + t.Helper() + interval, err := model.ParseDuration("10s") + require.NoError(t, err) + rules := apimodels.PostableRuleGroupConfig{ + Name: "arulegroup", + Interval: interval, + } + + // Create rules that will fire as quickly as possible for all the routes. + for _, alertName := range alertNames { + rules.Rules = append(rules.Rules, apimodels.PostableExtendedRuleNode{ + GrafanaManagedAlert: &apimodels.PostableGrafanaRule{ + Title: alertName, + Condition: "A", + Data: []ngmodels.AlertQuery{ + { + RefID: "A", + RelativeTimeRange: ngmodels.RelativeTimeRange{ + From: ngmodels.Duration(time.Duration(5) * time.Hour), + To: ngmodels.Duration(time.Duration(3) * time.Hour), + }, + DatasourceUID: "-100", + Model: json.RawMessage(`{ + "type": "math", + "expression": "2 + 3 > 1" + }`), + }, + }, + }, + }) + } + + b, err := json.Marshal(rules) + require.NoError(t, err) + + return string(b) +} + +func getRequest(t *testing.T, url string, expStatusCode int) *http.Response { + t.Helper() + // nolint:gosec + resp, err := http.Get(url) + t.Cleanup(func() { + require.NoError(t, resp.Body.Close()) + }) + require.NoError(t, err) + require.Equal(t, expStatusCode, resp.StatusCode) + return resp +} + +func postRequest(t *testing.T, url string, body string, expStatusCode int) { + t.Helper() + buf := bytes.NewReader([]byte(body)) + // nolint:gosec + resp, err := http.Post(url, "application/json", buf) + t.Cleanup(func() { + require.NoError(t, resp.Body.Close()) + }) + require.NoError(t, err) + require.Equal(t, expStatusCode, resp.StatusCode) +} + +type mockNotificationChannel struct { + t *testing.T + server *http.Server + + receivedNotifications map[string][]string + receivedNotificationsMtx sync.Mutex +} + +func newMockNotificationChannel(t *testing.T, grafanaListedAddr string) *mockNotificationChannel { + lastDigit := grafanaListedAddr[len(grafanaListedAddr)-1] - 48 + lastDigit = (lastDigit + 1) % 10 + newAddr := fmt.Sprintf("%s%01d", grafanaListedAddr[:len(grafanaListedAddr)-1], lastDigit) + + nc := &mockNotificationChannel{ + server: &http.Server{ + Addr: newAddr, + }, + receivedNotifications: make(map[string][]string), + t: t, + } + + nc.server.Handler = nc + go func() { + require.Equal(t, http.ErrServerClosed, nc.server.ListenAndServe()) + }() + + return nc +} + +func (nc *mockNotificationChannel) ServeHTTP(res http.ResponseWriter, req *http.Request) { + nc.t.Helper() + nc.receivedNotificationsMtx.Lock() + defer nc.receivedNotificationsMtx.Unlock() + + urlParts := strings.Split(req.URL.String(), "/") + key := fmt.Sprintf("%s/%s", urlParts[len(urlParts)-2], urlParts[len(urlParts)-1]) + body := getBody(nc.t, req.Body) + + nc.receivedNotifications[key] = append(nc.receivedNotifications[key], body) + res.WriteHeader(http.StatusOK) +} + +func getBody(t *testing.T, body io.ReadCloser) string { + t.Helper() + b, err := ioutil.ReadAll(body) + require.NoError(t, err) + return string(b) +} + +func (nc *mockNotificationChannel) totalNotifications() int { + total := 0 + nc.receivedNotificationsMtx.Lock() + defer nc.receivedNotificationsMtx.Unlock() + for _, v := range nc.receivedNotifications { + total += len(v) + } + return total +} + +func (nc *mockNotificationChannel) matchesExpNotifications(exp map[string][]string) bool { + nc.t.Helper() + nc.receivedNotificationsMtx.Lock() + defer nc.receivedNotificationsMtx.Unlock() + + if len(nc.receivedNotifications) != len(exp) { + return false + } + + for expKey, expVals := range exp { + actVals, ok := nc.receivedNotifications[expKey] + if !ok || len(actVals) != len(expVals) { + return false + } + for i := range expVals { + expVal := expVals[i] + var r *regexp.Regexp + switch expKey { + case "webhook_recv/webhook_test": + // It has a time component "startsAt". + r = regexp.MustCompile(`.*"startsAt"\s*:\s*"([^"]+)"`) + case "slack_recvX/slack_testX": + fallthrough + case "slack_recv1/slack_test_without_token": + // It has a time component "ts". + r = regexp.MustCompile(`.*"ts"\s*:\s*([0-9]{10})`) + case "pagerduty_recvX/pagerduty_testX": + // It has a changing "source". + r = regexp.MustCompile(`.*"source"\s*:\s*"([^"]+)"`) + } + if r != nil { + parts := r.FindStringSubmatch(actVals[i]) + require.Equal(nc.t, 2, len(parts)) + expVal = fmt.Sprintf(expVal, parts[1]) + } + + var expJson, actJson interface{} + require.NoError(nc.t, json.Unmarshal([]byte(expVal), &expJson)) + require.NoError(nc.t, json.Unmarshal([]byte(actVals[i]), &actJson)) + if !assert.ObjectsAreEqual(expJson, actJson) { + return false + } + } + } + + return true +} + +func (nc *mockNotificationChannel) Close() error { + return nc.server.Close() +} + +// alertmanagerConfig has the config for all the notification channels +// that we want to test. It is recommended to use different URL for each +// channel and have 1 route per channel. +// group_wait 0s means the notification is sent as soon as it is received. +const alertmanagerConfig = ` +{ + "alertmanager_config": { + "route": { + "receiver": "slack_recv1", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "routes": [ + { + "receiver": "email_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"EmailAlert\"" + ] + }, + { + "receiver": "slack_recv1", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"SlackAlert1\"" + ] + }, + { + "receiver": "slack_recv2", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"SlackAlert2\"" + ] + }, + { + "receiver": "pagerduty_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"PagerdutyAlert\"" + ] + }, + { + "receiver": "dingding_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"DingDingAlert\"" + ] + }, + { + "receiver": "teams_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"TeamsAlert\"" + ] + }, + { + "receiver": "webhook_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"WebhookAlert\"" + ] + }, + { + "receiver": "telegram_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"TelegramAlert\"" + ] + } + ] + }, + "receivers": [ + { + "name": "email_recv", + "grafana_managed_receiver_configs": [ + { + "name": "email_test", + "type": "email", + "settings": { + "addresses": "test@email.com", + "singleEmail": true + } + } + ] + }, + { + "name": "dingding_recv", + "grafana_managed_receiver_configs": [ + { + "name": "dingding_test", + "type": "dingding", + "settings": { + "url": "http://CHANNEL_ADDR/dingding_recv/dingding_test" + } + } + ] + }, + { + "name": "teams_recv", + "grafana_managed_receiver_configs": [ + { + "name": "teams_test", + "type": "teams", + "settings": { + "url": "http://CHANNEL_ADDR/teams_recv/teams_test" + } + } + ] + }, + { + "name": "webhook_recv", + "grafana_managed_receiver_configs": [ + { + "name": "webhook_test", + "type": "webhook", + "settings": { + "url": "http://CHANNEL_ADDR/webhook_recv/webhook_test", + "username": "my_username", + "httpMethod": "POST", + "maxAlerts": "5" + }, + "secureSettings": { + "password": "mysecretpassword" + } + } + ] + }, + { + "name": "telegram_recv", + "grafana_managed_receiver_configs": [ + { + "name": "telegram_test", + "type": "telegram", + "settings": { + "chatid": "telegram_chat_id" + }, + "secureSettings": { + "bottoken": "6sh027hs034h" + } + } + ] + }, + { + "name": "slack_recv1", + "grafana_managed_receiver_configs": [ + { + "name": "slack_test_without_token", + "type": "slack", + "settings": { + "recipient": "#test-channel", + "mentionChannel": "here", + "mentionUsers": "user1, user2", + "mentionGroups": "group1, group2", + "username": "Integration Test", + "icon_emoji": "🚀", + "icon_url": "https://awesomeemoji.com/rocket", + "text": "Integration Test {{ template \"slack.default.text\" . }}", + "title": "Integration Test {{ template \"slack.default.title\" . }}", + "fallback": "Integration Test {{ template \"slack.default.title\" . }}" + }, + "secureSettings": { + "url": "http://CHANNEL_ADDR/slack_recv1/slack_test_without_token" + } + } + ] + }, + { + "name": "slack_recv2", + "grafana_managed_receiver_configs": [ + { + "name": "slack_test_with_token", + "type": "slack", + "settings": { + "recipient": "#test-channel", + "mentionUsers": "user1, user2", + "username": "Integration Test" + }, + "secureSettings": { + "token": "myfullysecrettoken" + } + } + ] + }, + { + "name": "pagerduty_recv", + "grafana_managed_receiver_configs": [ + { + "name": "pagerduty_test", + "type": "pagerduty", + "settings": { + "severity": "warning", + "class": "testclass", + "component": "Integration Test", + "group": "testgroup", + "summary": "Integration Test {{ template \"pagerduty.default.description\" . }}" + }, + "secureSettings": { + "integrationKey": "pagerduty_recv/pagerduty_test" + } + } + ] + } + ] + } +} +` + +var expAlertmanagerConfigFromAPI = ` +{ + "template_files": null, + "alertmanager_config": { + "route": { + "receiver": "slack_recv1", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "routes": [ + { + "receiver": "email_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"EmailAlert\"" + ] + }, + { + "receiver": "slack_recv1", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"SlackAlert1\"" + ] + }, + { + "receiver": "slack_recv2", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"SlackAlert2\"" + ] + }, + { + "receiver": "pagerduty_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"PagerdutyAlert\"" + ] + }, + { + "receiver": "dingding_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"DingDingAlert\"" + ] + }, + { + "receiver": "teams_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"TeamsAlert\"" + ] + }, + { + "receiver": "webhook_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"WebhookAlert\"" + ] + }, + { + "receiver": "telegram_recv", + "group_wait": "0s", + "group_by": [ + "alertname" + ], + "matchers": [ + "alertname=\"TelegramAlert\"" + ] + } + ] + }, + "templates": null, + "receivers": [ + { + "name": "email_recv", + "grafana_managed_receiver_configs": [ + { + "id": 0, + "uid": "", + "name": "email_test", + "type": "email", + "isDefault": false, + "sendReminder": false, + "disableResolveMessage": false, + "frequency": "", + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "settings": { + "addresses": "test@email.com", + "singleEmail": true + }, + "secureFields": {} + } + ] + }, + { + "name": "dingding_recv", + "grafana_managed_receiver_configs": [ + { + "id": 0, + "uid": "", + "name": "dingding_test", + "type": "dingding", + "isDefault": false, + "sendReminder": false, + "disableResolveMessage": false, + "frequency": "", + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "settings": { + "url": "http://CHANNEL_ADDR/dingding_recv/dingding_test" + }, + "secureFields": {} + } + ] + }, + { + "name": "teams_recv", + "grafana_managed_receiver_configs": [ + { + "id": 0, + "uid": "", + "name": "teams_test", + "type": "teams", + "isDefault": false, + "sendReminder": false, + "disableResolveMessage": false, + "frequency": "", + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "settings": { + "url": "http://CHANNEL_ADDR/teams_recv/teams_test" + }, + "secureFields": {} + } + ] + }, + { + "name": "webhook_recv", + "grafana_managed_receiver_configs": [ + { + "id": 0, + "uid": "", + "name": "webhook_test", + "type": "webhook", + "isDefault": false, + "sendReminder": false, + "disableResolveMessage": false, + "frequency": "", + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "settings": { + "url": "http://CHANNEL_ADDR/webhook_recv/webhook_test", + "username": "my_username", + "httpMethod": "POST", + "maxAlerts": "5" + }, + "secureFields": { + "password": true + } + } + ] + }, + { + "name": "telegram_recv", + "grafana_managed_receiver_configs": [ + { + "id": 0, + "uid": "", + "name": "telegram_test", + "type": "telegram", + "isDefault": false, + "sendReminder": false, + "disableResolveMessage": false, + "frequency": "", + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "settings": { + "chatid": "telegram_chat_id" + }, + "secureFields": { + "bottoken": true + } + } + ] + }, + { + "name": "slack_recv1", + "grafana_managed_receiver_configs": [ + { + "id": 0, + "uid": "", + "name": "slack_test_without_token", + "type": "slack", + "isDefault": false, + "sendReminder": false, + "disableResolveMessage": false, + "frequency": "", + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "settings": { + "fallback": "Integration Test {{ template \"slack.default.title\" . }}", + "icon_emoji": "🚀", + "icon_url": "https://awesomeemoji.com/rocket", + "mentionChannel": "here", + "mentionGroups": "group1, group2", + "mentionUsers": "user1, user2", + "recipient": "#test-channel", + "text": "Integration Test {{ template \"slack.default.text\" . }}", + "title": "Integration Test {{ template \"slack.default.title\" . }}", + "username": "Integration Test" + }, + "secureFields": { + "url": true + } + } + ] + }, + { + "name": "slack_recv2", + "grafana_managed_receiver_configs": [ + { + "id": 0, + "uid": "", + "name": "slack_test_with_token", + "type": "slack", + "isDefault": false, + "sendReminder": false, + "disableResolveMessage": false, + "frequency": "", + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "settings": { + "mentionUsers": "user1, user2", + "recipient": "#test-channel", + "username": "Integration Test" + }, + "secureFields": { + "token": true + } + } + ] + }, + { + "name": "pagerduty_recv", + "grafana_managed_receiver_configs": [ + { + "id": 0, + "uid": "", + "name": "pagerduty_test", + "type": "pagerduty", + "isDefault": false, + "sendReminder": false, + "disableResolveMessage": false, + "frequency": "", + "created": "0001-01-01T00:00:00Z", + "updated": "0001-01-01T00:00:00Z", + "settings": { + "class": "testclass", + "component": "Integration Test", + "group": "testgroup", + "severity": "warning", + "summary": "Integration Test {{ template \"pagerduty.default.description\" . }}" + }, + "secureFields": { + "integrationKey": true + } + } + ] + } + ] + } +} +` + +// expNotifications is all the expected notifications. +// The key for the map is taken from the URL. The last 2 components of URL +// split with "/" forms the key for that route. +var expNotifications = map[string][]string{ + "slack_recv1/slack_test_without_token": { + `{ + "channel": "#test-channel", + "username": "Integration Test", + "icon_emoji": "🚀", + "icon_url": "https://awesomeemoji.com/rocket", + "attachments": [ + { + "title": "Integration Test [FIRING:1] (SlackAlert1 UID_SlackAlert1)", + "title_link": "http:/localhost:3000/alerting/list", + "text": "Integration Test ", + "fallback": "Integration Test [FIRING:1] (SlackAlert1 UID_SlackAlert1)", + "footer": "Grafana v", + "footer_icon": "https://grafana.com/assets/img/fav32.png", + "color": "#D63232", + "ts": %s + } + ], + "blocks": [ + { + "text": { + "text": " <@user1><@user2>", + "type": "mrkdwn" + }, + "type": "section" + } + ] + }`, + }, + "slack_recvX/slack_testX": { + `{ + "channel": "#test-channel", + "username": "Integration Test", + "attachments": [ + { + "title": "[FIRING:1] (SlackAlert2 UID_SlackAlert2)", + "title_link": "http:/localhost:3000/alerting/list", + "text": "\n**Firing**\nLabels:\n - alertname = SlackAlert2\n - __alert_rule_uid__ = UID_SlackAlert2\nAnnotations:\nSource: \n\n\n\n\n", + "fallback": "[FIRING:1] (SlackAlert2 UID_SlackAlert2)", + "footer": "Grafana v", + "footer_icon": "https://grafana.com/assets/img/fav32.png", + "color": "#D63232", + "ts": %s + } + ], + "blocks": [ + { + "text": { + "text": "<@user1><@user2>", + "type": "mrkdwn" + }, + "type": "section" + } + ] + }`, + }, + "pagerduty_recvX/pagerduty_testX": { + `{ + "routing_key": "pagerduty_recv/pagerduty_test", + "dedup_key": "718643b9694d44f7f2b21458afd1b079cb403cf264e51894ff3c9745238bcced", + "description": "[firing:1] (PagerdutyAlert UID_PagerdutyAlert)", + "event_action": "trigger", + "payload": { + "summary": "Integration Test [FIRING:1] (PagerdutyAlert UID_PagerdutyAlert)", + "source": "%s", + "severity": "warning", + "class": "testclass", + "component": "Integration Test", + "group": "testgroup", + "custom_details": { + "firing": "Labels:\n - alertname = PagerdutyAlert\n - __alert_rule_uid__ = UID_PagerdutyAlert\nAnnotations:\nSource: \n", + "num_firing": "1", + "num_resolved": "0", + "resolved": "" + } + }, + "client": "Grafana", + "client_url": "http://localhost:3000/", + "links": [ + { + "href": "http://localhost:3000/", + "text": "External URL" + } + ] + }`, + }, + "dingding_recv/dingding_test": { + `{ + "link": { + "messageUrl": "dingtalk://dingtalkclient/page/link?pc_slide=false&url=http%3A%2Flocalhost%3A3000%2Falerting%2Flist", + "text": "\n**Firing**\nLabels:\n - alertname = DingDingAlert\n - __alert_rule_uid__ = UID_DingDingAlert\nAnnotations:\nSource: \n\n\n\n\n", + "title": "[firing:1] (DingDingAlert UID_DingDingAlert)" + }, + "msgtype": "link" + }`, + }, + "teams_recv/teams_test": { + `{ + "@context": "http://schema.org/extensions", + "@type": "MessageCard", + "potentialAction": [ + { + "@context": "http://schema.org", + "@type": "OpenUri", + "name": "View Rule", + "targets": [ + { + "os": "default", + "uri": "http:/localhost:3000/alerting/list" + } + ] + } + ], + "sections": [ + { + "text": "\n**Firing**\nLabels:\n - alertname = TeamsAlert\n - __alert_rule_uid__ = UID_TeamsAlert\nAnnotations:\nSource: \n\n\n\n\n", + "title": "Details" + } + ], + "summary": "[firing:1] (TeamsAlert UID_TeamsAlert)", + "themeColor": "#D63232", + "title": "[firing:1] (TeamsAlert UID_TeamsAlert)" + }`, + }, + "webhook_recv/webhook_test": { + `{ + "receiver": "webhook_recv", + "status": "firing", + "alerts": [ + { + "status": "firing", + "labels": { + "__alert_rule_uid__": "UID_WebhookAlert", + "alertname": "WebhookAlert" + }, + "annotations": {}, + "startsAt": "%s", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "", + "fingerprint": "929467973978d053" + } + ], + "groupLabels": {}, + "commonLabels": { + "__alert_rule_uid__": "UID_WebhookAlert", + "alertname": "WebhookAlert" + }, + "commonAnnotations": {}, + "externalURL": "http://localhost:3000/", + "version": "1", + "groupKey": "{}/{alertname=\"WebhookAlert\"}:{}", + "truncatedAlerts": 0, + "title": "[FIRING:1] (WebhookAlert UID_WebhookAlert)", + "state": "alerting", + "message": "\n**Firing**\nLabels:\n - alertname = WebhookAlert\n - __alert_rule_uid__ = UID_WebhookAlert\nAnnotations:\nSource: \n\n\n\n\n" + }`, + }, +} diff --git a/pkg/tests/testinfra/testinfra.go b/pkg/tests/testinfra/testinfra.go index 77fab42cdfb..efadef24cf2 100644 --- a/pkg/tests/testinfra/testinfra.go +++ b/pkg/tests/testinfra/testinfra.go @@ -202,6 +202,11 @@ func CreateGrafDir(t *testing.T, opts ...GrafanaOpts) (string, string) { _, err = anonSect.NewKey("enabled", "true") require.NoError(t, err) + alertingSect, err := cfg.NewSection("alerting") + require.NoError(t, err) + _, err = alertingSect.NewKey("notification_timeout_seconds", "1") + require.NoError(t, err) + for _, o := range opts { if o.EnableCSP { securitySect, err := cfg.NewSection("security")