mirror of
https://github.com/grafana/grafana.git
synced 2025-02-11 08:05:43 -06:00
NGAlert: Add integration tests for notification channels (#33431)
* NGAlert: Add integration tests for notification channels Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Fix the failing tests Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Fix review comments Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com> * Override creation of rule UID, remove only namespace UID Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>
This commit is contained in:
Ganesh Vernekar
GitHub
parent
1a649afdd8
commit
ec3214bac2
@ -446,7 +446,7 @@ func (am *Alertmanager) PutAlerts(postableAlerts apimodels.PostableAlerts) error
|
||||
UpdatedAt: now,
|
||||
}
|
||||
for k, v := range a.Labels {
|
||||
if len(v) == 0 { // Skip empty labels.
|
||||
if len(v) == 0 || k == ngmodels.NamespaceUIDLabel { // Skip empty and namespace UID labels.
|
||||
continue
|
||||
}
|
||||
alert.Alert.Labels[model.LabelName(k)] = model.LabelValue(v)
|
||||
|
||||
@ -25,7 +25,7 @@ const (
|
||||
)
|
||||
|
||||
var (
|
||||
pagerdutyEventAPIURL = "https://events.pagerduty.com/v2/enqueue"
|
||||
PagerdutyEventAPIURL = "https://events.pagerduty.com/v2/enqueue"
|
||||
)
|
||||
|
||||
// PagerdutyNotifier is responsible for sending
|
||||
@ -93,7 +93,7 @@ func (pn *PagerdutyNotifier) Notify(ctx context.Context, as ...*types.Alert) (bo
|
||||
|
||||
pn.log.Info("Notifying Pagerduty", "event_type", eventType)
|
||||
cmd := &models.SendWebhookSync{
|
||||
Url: pagerdutyEventAPIURL,
|
||||
Url: PagerdutyEventAPIURL,
|
||||
Body: string(body),
|
||||
HttpMethod: "POST",
|
||||
HttpHeader: map[string]string{
|
||||
|
||||
@ -50,7 +50,7 @@ type SlackNotifier struct {
|
||||
|
||||
var reRecipient *regexp.Regexp = regexp.MustCompile("^((@[a-z0-9][a-zA-Z0-9._-]*)|(#[^ .A-Z]{1,79})|([a-zA-Z0-9]+))$")
|
||||
|
||||
const slackAPIEndpoint = "https://slack.com/api/chat.postMessage"
|
||||
var SlackAPIEndpoint = "https://slack.com/api/chat.postMessage"
|
||||
|
||||
// NewSlackNotifier is the constructor for the Slack notifier
|
||||
func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*SlackNotifier, error) {
|
||||
@ -60,7 +60,7 @@ func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*S
|
||||
|
||||
slackURL := model.DecryptedValue("url", model.Settings.Get("url").MustString())
|
||||
if slackURL == "" {
|
||||
slackURL = slackAPIEndpoint
|
||||
slackURL = SlackAPIEndpoint
|
||||
}
|
||||
apiURL, err := url.Parse(slackURL)
|
||||
if err != nil {
|
||||
@ -72,7 +72,7 @@ func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*S
|
||||
if !reRecipient.MatchString(recipient) {
|
||||
return nil, alerting.ValidationError{Reason: fmt.Sprintf("recipient on invalid format: %q", recipient)}
|
||||
}
|
||||
} else if apiURL.String() == slackAPIEndpoint {
|
||||
} else if apiURL.String() == SlackAPIEndpoint {
|
||||
return nil, alerting.ValidationError{
|
||||
Reason: "recipient must be specified when using the Slack chat API",
|
||||
}
|
||||
@ -104,7 +104,7 @@ func NewSlackNotifier(model *models.AlertNotification, t *template.Template) (*S
|
||||
}
|
||||
|
||||
token := model.DecryptedValue("token", model.Settings.Get("token").MustString())
|
||||
if token == "" && apiURL.String() == slackAPIEndpoint {
|
||||
if token == "" && apiURL.String() == SlackAPIEndpoint {
|
||||
return nil, alerting.ValidationError{
|
||||
Reason: "token must be specified when using the Slack chat API",
|
||||
}
|
||||
@ -172,7 +172,7 @@ func (sn *SlackNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
|
||||
request.Header.Set("Content-Type", "application/json")
|
||||
request.Header.Set("User-Agent", "Grafana")
|
||||
if sn.Token == "" {
|
||||
if sn.URL.String() == slackAPIEndpoint {
|
||||
if sn.URL.String() == SlackAPIEndpoint {
|
||||
panic("Token should be set when using the Slack chat API")
|
||||
}
|
||||
} else {
|
||||
|
||||
@ -18,8 +18,8 @@ import (
|
||||
old_notifiers "github.com/grafana/grafana/pkg/services/alerting/notifiers"
|
||||
)
|
||||
|
||||
const (
|
||||
telegramAPIURL = "https://api.telegram.org/bot%s/sendMessage"
|
||||
var (
|
||||
TelegramAPIURL = "https://api.telegram.org/bot%s/sendMessage"
|
||||
)
|
||||
|
||||
// TelegramNotifier is responsible for sending
|
||||
@ -90,7 +90,7 @@ func (tn *TelegramNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo
|
||||
|
||||
tn.log.Info("sending telegram notification", "chat_id", tn.ChatID)
|
||||
cmd := &models.SendWebhookSync{
|
||||
Url: fmt.Sprintf(telegramAPIURL, tn.BotToken),
|
||||
Url: fmt.Sprintf(TelegramAPIURL, tn.BotToken),
|
||||
Body: body.String(),
|
||||
HttpMethod: "POST",
|
||||
HttpHeader: map[string]string{
|
||||
|
||||
@ -203,7 +203,7 @@ func (st DBstore) UpsertAlertRules(rules []UpsertRule) error {
|
||||
var parentVersion int64
|
||||
switch r.Existing {
|
||||
case nil: // new rule
|
||||
uid, err := generateNewAlertRuleUID(sess, r.New.OrgID)
|
||||
uid, err := GenerateNewAlertRuleUID(sess, r.New.OrgID, r.New.Title)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to generate UID for alert rule %q: %w", r.New.Title, err)
|
||||
}
|
||||
@ -411,7 +411,10 @@ func (st DBstore) GetAlertRulesForScheduling(query *ngmodels.ListAlertRulesQuery
|
||||
})
|
||||
}
|
||||
|
||||
func generateNewAlertRuleUID(sess *sqlstore.DBSession, orgID int64) (string, error) {
|
||||
// GenerateNewAlertRuleUID generates a unique UID for a rule.
|
||||
// This is set as a variable so that the tests can override it.
|
||||
// The ruleTitle is only used by the mocked functions.
|
||||
var GenerateNewAlertRuleUID = func(sess *sqlstore.DBSession, orgID int64, ruleTitle string) (string, error) {
|
||||
for i := 0; i < 3; i++ {
|
||||
uid := util.GenerateShortUID()
|
||||
|
||||
|
||||
@ -8,6 +8,7 @@ import (
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/tests/testinfra"
|
||||
)
|
||||
@ -15,13 +16,17 @@ import (
|
||||
func TestAvailableChannels(t *testing.T) {
|
||||
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
||||
EnableFeatureToggles: []string{"ngalert"},
|
||||
AnonymousUserRole: models.ROLE_EDITOR,
|
||||
DisableAnonymous: true,
|
||||
})
|
||||
|
||||
store := testinfra.SetUpDatabase(t, dir)
|
||||
store.Bus = bus.GetBus()
|
||||
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
|
||||
|
||||
alertsURL := fmt.Sprintf("http://%s/api/alert-notifiers", grafanaListedAddr)
|
||||
// Create a user to make authenticated requests
|
||||
require.NoError(t, createUser(t, store, models.ROLE_EDITOR, "grafana", "password"))
|
||||
|
||||
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/alert-notifiers", grafanaListedAddr)
|
||||
// nolint:gosec
|
||||
resp, err := http.Get(alertsURL)
|
||||
require.NoError(t, err)
|
||||
|
||||
976
pkg/tests/api/alerting/api_notification_channel_test.go
Normal file
976
pkg/tests/api/alerting/api_notification_channel_test.go
Normal file
@ -0,0 +1,976 @@
|
||||
package alerting
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/prometheus/common/model"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/bus"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/notifier/channels"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/store"
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore"
|
||||
"github.com/grafana/grafana/pkg/tests/testinfra"
|
||||
)
|
||||
|
||||
func TestNotificationChannels(t *testing.T) {
|
||||
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
|
||||
EnableFeatureToggles: []string{"ngalert"},
|
||||
DisableAnonymous: true,
|
||||
})
|
||||
|
||||
s := testinfra.SetUpDatabase(t, dir)
|
||||
s.Bus = bus.GetBus()
|
||||
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, s)
|
||||
|
||||
mockChannel := newMockNotificationChannel(t, grafanaListedAddr)
|
||||
amConfig := getAlertmanagerConfig(mockChannel.server.Addr)
|
||||
|
||||
// Overriding some URLs to send to the mock channel.
|
||||
channels.SlackAPIEndpoint = fmt.Sprintf("http://%s/slack_recvX/slack_testX", mockChannel.server.Addr)
|
||||
channels.PagerdutyEventAPIURL = fmt.Sprintf("http://%s/pagerduty_recvX/pagerduty_testX", mockChannel.server.Addr)
|
||||
channels.TelegramAPIURL = fmt.Sprintf("http://%s/telegram_recv/bot%%s", mockChannel.server.Addr)
|
||||
|
||||
// Create a user to make authenticated requests
|
||||
require.NoError(t, createUser(t, s, models.ROLE_EDITOR, "grafana", "password"))
|
||||
|
||||
{
|
||||
// There are no notification channel config initially.
|
||||
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr)
|
||||
_ = getRequest(t, alertsURL, http.StatusNotFound) // nolint
|
||||
}
|
||||
|
||||
{
|
||||
// Create the namespace we'll save our alerts to.
|
||||
require.NoError(t, createFolder(t, s, 0, "default"))
|
||||
|
||||
// Post the alertmanager config.
|
||||
u := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr)
|
||||
postRequest(t, u, amConfig, http.StatusAccepted)
|
||||
|
||||
// Verifying that all the receivers and routes have been registered.
|
||||
alertsURL := fmt.Sprintf("http://grafana:password@%s/api/alertmanager/grafana/config/api/v1/alerts", grafanaListedAddr)
|
||||
resp := getRequest(t, alertsURL, http.StatusOK) // nolint
|
||||
b := getBody(t, resp.Body)
|
||||
require.JSONEq(t, getExpAlertmanagerConfigFromAPI(mockChannel.server.Addr), b)
|
||||
}
|
||||
|
||||
{
|
||||
// Create rules that will fire as quickly as possible
|
||||
|
||||
originalFunction := store.GenerateNewAlertRuleUID
|
||||
t.Cleanup(func() {
|
||||
store.GenerateNewAlertRuleUID = originalFunction
|
||||
})
|
||||
store.GenerateNewAlertRuleUID = func(_ *sqlstore.DBSession, _ int64, ruleTitle string) (string, error) {
|
||||
return "UID_" + ruleTitle, nil
|
||||
}
|
||||
|
||||
rulesConfig := getRulesConfig(t)
|
||||
u := fmt.Sprintf("http://grafana:password@%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr)
|
||||
postRequest(t, u, rulesConfig, http.StatusAccepted)
|
||||
}
|
||||
|
||||
// Eventually, we'll get all the desired alerts.
|
||||
// nolint:gosec
|
||||
require.Eventually(t, func() bool {
|
||||
return mockChannel.totalNotifications() == len(alertNames) &&
|
||||
mockChannel.matchesExpNotifications(expNotifications)
|
||||
}, 25*time.Second, 1*time.Second)
|
||||
|
||||
require.NoError(t, mockChannel.Close())
|
||||
}
|
||||
|
||||
func getAlertmanagerConfig(channelAddr string) string {
|
||||
return strings.ReplaceAll(alertmanagerConfig, "CHANNEL_ADDR", channelAddr)
|
||||
}
|
||||
|
||||
func getExpAlertmanagerConfigFromAPI(channelAddr string) string {
|
||||
return strings.ReplaceAll(expAlertmanagerConfigFromAPI, "CHANNEL_ADDR", channelAddr)
|
||||
}
|
||||
|
||||
// alertNames are name of alerts to be sent. This should be in sync with
|
||||
// the routes that we define in Alertmanager config.
|
||||
// EmailAlert and TelegramAlert are missing because they don't
|
||||
// send a JSON. Email and POST body are yet to be supported in the tests.
|
||||
var alertNames = []string{"DingDingAlert", "SlackAlert1", "SlackAlert2", "PagerdutyAlert", "TeamsAlert", "WebhookAlert"}
|
||||
|
||||
func getRulesConfig(t *testing.T) string {
|
||||
t.Helper()
|
||||
interval, err := model.ParseDuration("10s")
|
||||
require.NoError(t, err)
|
||||
rules := apimodels.PostableRuleGroupConfig{
|
||||
Name: "arulegroup",
|
||||
Interval: interval,
|
||||
}
|
||||
|
||||
// Create rules that will fire as quickly as possible for all the routes.
|
||||
for _, alertName := range alertNames {
|
||||
rules.Rules = append(rules.Rules, apimodels.PostableExtendedRuleNode{
|
||||
GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
|
||||
Title: alertName,
|
||||
Condition: "A",
|
||||
Data: []ngmodels.AlertQuery{
|
||||
{
|
||||
RefID: "A",
|
||||
RelativeTimeRange: ngmodels.RelativeTimeRange{
|
||||
From: ngmodels.Duration(time.Duration(5) * time.Hour),
|
||||
To: ngmodels.Duration(time.Duration(3) * time.Hour),
|
||||
},
|
||||
DatasourceUID: "-100",
|
||||
Model: json.RawMessage(`{
|
||||
"type": "math",
|
||||
"expression": "2 + 3 > 1"
|
||||
}`),
|
||||
},
|
||||
},
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
b, err := json.Marshal(rules)
|
||||
require.NoError(t, err)
|
||||
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func getRequest(t *testing.T, url string, expStatusCode int) *http.Response {
|
||||
t.Helper()
|
||||
// nolint:gosec
|
||||
resp, err := http.Get(url)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, resp.Body.Close())
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expStatusCode, resp.StatusCode)
|
||||
return resp
|
||||
}
|
||||
|
||||
func postRequest(t *testing.T, url string, body string, expStatusCode int) {
|
||||
t.Helper()
|
||||
buf := bytes.NewReader([]byte(body))
|
||||
// nolint:gosec
|
||||
resp, err := http.Post(url, "application/json", buf)
|
||||
t.Cleanup(func() {
|
||||
require.NoError(t, resp.Body.Close())
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expStatusCode, resp.StatusCode)
|
||||
}
|
||||
|
||||
type mockNotificationChannel struct {
|
||||
t *testing.T
|
||||
server *http.Server
|
||||
|
||||
receivedNotifications map[string][]string
|
||||
receivedNotificationsMtx sync.Mutex
|
||||
}
|
||||
|
||||
func newMockNotificationChannel(t *testing.T, grafanaListedAddr string) *mockNotificationChannel {
|
||||
lastDigit := grafanaListedAddr[len(grafanaListedAddr)-1] - 48
|
||||
lastDigit = (lastDigit + 1) % 10
|
||||
newAddr := fmt.Sprintf("%s%01d", grafanaListedAddr[:len(grafanaListedAddr)-1], lastDigit)
|
||||
|
||||
nc := &mockNotificationChannel{
|
||||
server: &http.Server{
|
||||
Addr: newAddr,
|
||||
},
|
||||
receivedNotifications: make(map[string][]string),
|
||||
t: t,
|
||||
}
|
||||
|
||||
nc.server.Handler = nc
|
||||
go func() {
|
||||
require.Equal(t, http.ErrServerClosed, nc.server.ListenAndServe())
|
||||
}()
|
||||
|
||||
return nc
|
||||
}
|
||||
|
||||
func (nc *mockNotificationChannel) ServeHTTP(res http.ResponseWriter, req *http.Request) {
|
||||
nc.t.Helper()
|
||||
nc.receivedNotificationsMtx.Lock()
|
||||
defer nc.receivedNotificationsMtx.Unlock()
|
||||
|
||||
urlParts := strings.Split(req.URL.String(), "/")
|
||||
key := fmt.Sprintf("%s/%s", urlParts[len(urlParts)-2], urlParts[len(urlParts)-1])
|
||||
body := getBody(nc.t, req.Body)
|
||||
|
||||
nc.receivedNotifications[key] = append(nc.receivedNotifications[key], body)
|
||||
res.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func getBody(t *testing.T, body io.ReadCloser) string {
|
||||
t.Helper()
|
||||
b, err := ioutil.ReadAll(body)
|
||||
require.NoError(t, err)
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func (nc *mockNotificationChannel) totalNotifications() int {
|
||||
total := 0
|
||||
nc.receivedNotificationsMtx.Lock()
|
||||
defer nc.receivedNotificationsMtx.Unlock()
|
||||
for _, v := range nc.receivedNotifications {
|
||||
total += len(v)
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
func (nc *mockNotificationChannel) matchesExpNotifications(exp map[string][]string) bool {
|
||||
nc.t.Helper()
|
||||
nc.receivedNotificationsMtx.Lock()
|
||||
defer nc.receivedNotificationsMtx.Unlock()
|
||||
|
||||
if len(nc.receivedNotifications) != len(exp) {
|
||||
return false
|
||||
}
|
||||
|
||||
for expKey, expVals := range exp {
|
||||
actVals, ok := nc.receivedNotifications[expKey]
|
||||
if !ok || len(actVals) != len(expVals) {
|
||||
return false
|
||||
}
|
||||
for i := range expVals {
|
||||
expVal := expVals[i]
|
||||
var r *regexp.Regexp
|
||||
switch expKey {
|
||||
case "webhook_recv/webhook_test":
|
||||
// It has a time component "startsAt".
|
||||
r = regexp.MustCompile(`.*"startsAt"\s*:\s*"([^"]+)"`)
|
||||
case "slack_recvX/slack_testX":
|
||||
fallthrough
|
||||
case "slack_recv1/slack_test_without_token":
|
||||
// It has a time component "ts".
|
||||
r = regexp.MustCompile(`.*"ts"\s*:\s*([0-9]{10})`)
|
||||
case "pagerduty_recvX/pagerduty_testX":
|
||||
// It has a changing "source".
|
||||
r = regexp.MustCompile(`.*"source"\s*:\s*"([^"]+)"`)
|
||||
}
|
||||
if r != nil {
|
||||
parts := r.FindStringSubmatch(actVals[i])
|
||||
require.Equal(nc.t, 2, len(parts))
|
||||
expVal = fmt.Sprintf(expVal, parts[1])
|
||||
}
|
||||
|
||||
var expJson, actJson interface{}
|
||||
require.NoError(nc.t, json.Unmarshal([]byte(expVal), &expJson))
|
||||
require.NoError(nc.t, json.Unmarshal([]byte(actVals[i]), &actJson))
|
||||
if !assert.ObjectsAreEqual(expJson, actJson) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func (nc *mockNotificationChannel) Close() error {
|
||||
return nc.server.Close()
|
||||
}
|
||||
|
||||
// alertmanagerConfig has the config for all the notification channels
|
||||
// that we want to test. It is recommended to use different URL for each
|
||||
// channel and have 1 route per channel.
|
||||
// group_wait 0s means the notification is sent as soon as it is received.
|
||||
const alertmanagerConfig = `
|
||||
{
|
||||
"alertmanager_config": {
|
||||
"route": {
|
||||
"receiver": "slack_recv1",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"routes": [
|
||||
{
|
||||
"receiver": "email_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"EmailAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "slack_recv1",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"SlackAlert1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "slack_recv2",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"SlackAlert2\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "pagerduty_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"PagerdutyAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "dingding_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"DingDingAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "teams_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"TeamsAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "webhook_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"WebhookAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "telegram_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"TelegramAlert\""
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"receivers": [
|
||||
{
|
||||
"name": "email_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"name": "email_test",
|
||||
"type": "email",
|
||||
"settings": {
|
||||
"addresses": "test@email.com",
|
||||
"singleEmail": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "dingding_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"name": "dingding_test",
|
||||
"type": "dingding",
|
||||
"settings": {
|
||||
"url": "http://CHANNEL_ADDR/dingding_recv/dingding_test"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "teams_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"name": "teams_test",
|
||||
"type": "teams",
|
||||
"settings": {
|
||||
"url": "http://CHANNEL_ADDR/teams_recv/teams_test"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "webhook_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"name": "webhook_test",
|
||||
"type": "webhook",
|
||||
"settings": {
|
||||
"url": "http://CHANNEL_ADDR/webhook_recv/webhook_test",
|
||||
"username": "my_username",
|
||||
"httpMethod": "POST",
|
||||
"maxAlerts": "5"
|
||||
},
|
||||
"secureSettings": {
|
||||
"password": "mysecretpassword"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "telegram_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"name": "telegram_test",
|
||||
"type": "telegram",
|
||||
"settings": {
|
||||
"chatid": "telegram_chat_id"
|
||||
},
|
||||
"secureSettings": {
|
||||
"bottoken": "6sh027hs034h"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slack_recv1",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"name": "slack_test_without_token",
|
||||
"type": "slack",
|
||||
"settings": {
|
||||
"recipient": "#test-channel",
|
||||
"mentionChannel": "here",
|
||||
"mentionUsers": "user1, user2",
|
||||
"mentionGroups": "group1, group2",
|
||||
"username": "Integration Test",
|
||||
"icon_emoji": "🚀",
|
||||
"icon_url": "https://awesomeemoji.com/rocket",
|
||||
"text": "Integration Test {{ template \"slack.default.text\" . }}",
|
||||
"title": "Integration Test {{ template \"slack.default.title\" . }}",
|
||||
"fallback": "Integration Test {{ template \"slack.default.title\" . }}"
|
||||
},
|
||||
"secureSettings": {
|
||||
"url": "http://CHANNEL_ADDR/slack_recv1/slack_test_without_token"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slack_recv2",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"name": "slack_test_with_token",
|
||||
"type": "slack",
|
||||
"settings": {
|
||||
"recipient": "#test-channel",
|
||||
"mentionUsers": "user1, user2",
|
||||
"username": "Integration Test"
|
||||
},
|
||||
"secureSettings": {
|
||||
"token": "myfullysecrettoken"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "pagerduty_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"name": "pagerduty_test",
|
||||
"type": "pagerduty",
|
||||
"settings": {
|
||||
"severity": "warning",
|
||||
"class": "testclass",
|
||||
"component": "Integration Test",
|
||||
"group": "testgroup",
|
||||
"summary": "Integration Test {{ template \"pagerduty.default.description\" . }}"
|
||||
},
|
||||
"secureSettings": {
|
||||
"integrationKey": "pagerduty_recv/pagerduty_test"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
var expAlertmanagerConfigFromAPI = `
|
||||
{
|
||||
"template_files": null,
|
||||
"alertmanager_config": {
|
||||
"route": {
|
||||
"receiver": "slack_recv1",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"routes": [
|
||||
{
|
||||
"receiver": "email_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"EmailAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "slack_recv1",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"SlackAlert1\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "slack_recv2",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"SlackAlert2\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "pagerduty_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"PagerdutyAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "dingding_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"DingDingAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "teams_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"TeamsAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "webhook_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"WebhookAlert\""
|
||||
]
|
||||
},
|
||||
{
|
||||
"receiver": "telegram_recv",
|
||||
"group_wait": "0s",
|
||||
"group_by": [
|
||||
"alertname"
|
||||
],
|
||||
"matchers": [
|
||||
"alertname=\"TelegramAlert\""
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"templates": null,
|
||||
"receivers": [
|
||||
{
|
||||
"name": "email_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"id": 0,
|
||||
"uid": "",
|
||||
"name": "email_test",
|
||||
"type": "email",
|
||||
"isDefault": false,
|
||||
"sendReminder": false,
|
||||
"disableResolveMessage": false,
|
||||
"frequency": "",
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"updated": "0001-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"addresses": "test@email.com",
|
||||
"singleEmail": true
|
||||
},
|
||||
"secureFields": {}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "dingding_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"id": 0,
|
||||
"uid": "",
|
||||
"name": "dingding_test",
|
||||
"type": "dingding",
|
||||
"isDefault": false,
|
||||
"sendReminder": false,
|
||||
"disableResolveMessage": false,
|
||||
"frequency": "",
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"updated": "0001-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"url": "http://CHANNEL_ADDR/dingding_recv/dingding_test"
|
||||
},
|
||||
"secureFields": {}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "teams_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"id": 0,
|
||||
"uid": "",
|
||||
"name": "teams_test",
|
||||
"type": "teams",
|
||||
"isDefault": false,
|
||||
"sendReminder": false,
|
||||
"disableResolveMessage": false,
|
||||
"frequency": "",
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"updated": "0001-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"url": "http://CHANNEL_ADDR/teams_recv/teams_test"
|
||||
},
|
||||
"secureFields": {}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "webhook_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"id": 0,
|
||||
"uid": "",
|
||||
"name": "webhook_test",
|
||||
"type": "webhook",
|
||||
"isDefault": false,
|
||||
"sendReminder": false,
|
||||
"disableResolveMessage": false,
|
||||
"frequency": "",
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"updated": "0001-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"url": "http://CHANNEL_ADDR/webhook_recv/webhook_test",
|
||||
"username": "my_username",
|
||||
"httpMethod": "POST",
|
||||
"maxAlerts": "5"
|
||||
},
|
||||
"secureFields": {
|
||||
"password": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "telegram_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"id": 0,
|
||||
"uid": "",
|
||||
"name": "telegram_test",
|
||||
"type": "telegram",
|
||||
"isDefault": false,
|
||||
"sendReminder": false,
|
||||
"disableResolveMessage": false,
|
||||
"frequency": "",
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"updated": "0001-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"chatid": "telegram_chat_id"
|
||||
},
|
||||
"secureFields": {
|
||||
"bottoken": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slack_recv1",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"id": 0,
|
||||
"uid": "",
|
||||
"name": "slack_test_without_token",
|
||||
"type": "slack",
|
||||
"isDefault": false,
|
||||
"sendReminder": false,
|
||||
"disableResolveMessage": false,
|
||||
"frequency": "",
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"updated": "0001-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"fallback": "Integration Test {{ template \"slack.default.title\" . }}",
|
||||
"icon_emoji": "🚀",
|
||||
"icon_url": "https://awesomeemoji.com/rocket",
|
||||
"mentionChannel": "here",
|
||||
"mentionGroups": "group1, group2",
|
||||
"mentionUsers": "user1, user2",
|
||||
"recipient": "#test-channel",
|
||||
"text": "Integration Test {{ template \"slack.default.text\" . }}",
|
||||
"title": "Integration Test {{ template \"slack.default.title\" . }}",
|
||||
"username": "Integration Test"
|
||||
},
|
||||
"secureFields": {
|
||||
"url": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "slack_recv2",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"id": 0,
|
||||
"uid": "",
|
||||
"name": "slack_test_with_token",
|
||||
"type": "slack",
|
||||
"isDefault": false,
|
||||
"sendReminder": false,
|
||||
"disableResolveMessage": false,
|
||||
"frequency": "",
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"updated": "0001-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"mentionUsers": "user1, user2",
|
||||
"recipient": "#test-channel",
|
||||
"username": "Integration Test"
|
||||
},
|
||||
"secureFields": {
|
||||
"token": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "pagerduty_recv",
|
||||
"grafana_managed_receiver_configs": [
|
||||
{
|
||||
"id": 0,
|
||||
"uid": "",
|
||||
"name": "pagerduty_test",
|
||||
"type": "pagerduty",
|
||||
"isDefault": false,
|
||||
"sendReminder": false,
|
||||
"disableResolveMessage": false,
|
||||
"frequency": "",
|
||||
"created": "0001-01-01T00:00:00Z",
|
||||
"updated": "0001-01-01T00:00:00Z",
|
||||
"settings": {
|
||||
"class": "testclass",
|
||||
"component": "Integration Test",
|
||||
"group": "testgroup",
|
||||
"severity": "warning",
|
||||
"summary": "Integration Test {{ template \"pagerduty.default.description\" . }}"
|
||||
},
|
||||
"secureFields": {
|
||||
"integrationKey": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
`
|
||||
|
||||
// expNotifications is all the expected notifications.
|
||||
// The key for the map is taken from the URL. The last 2 components of URL
|
||||
// split with "/" forms the key for that route.
|
||||
var expNotifications = map[string][]string{
|
||||
"slack_recv1/slack_test_without_token": {
|
||||
`{
|
||||
"channel": "#test-channel",
|
||||
"username": "Integration Test",
|
||||
"icon_emoji": "🚀",
|
||||
"icon_url": "https://awesomeemoji.com/rocket",
|
||||
"attachments": [
|
||||
{
|
||||
"title": "Integration Test [FIRING:1] (SlackAlert1 UID_SlackAlert1)",
|
||||
"title_link": "http:/localhost:3000/alerting/list",
|
||||
"text": "Integration Test ",
|
||||
"fallback": "Integration Test [FIRING:1] (SlackAlert1 UID_SlackAlert1)",
|
||||
"footer": "Grafana v",
|
||||
"footer_icon": "https://grafana.com/assets/img/fav32.png",
|
||||
"color": "#D63232",
|
||||
"ts": %s
|
||||
}
|
||||
],
|
||||
"blocks": [
|
||||
{
|
||||
"text": {
|
||||
"text": "<!here|here> <!subteam^group1><!subteam^group2> <@user1><@user2>",
|
||||
"type": "mrkdwn"
|
||||
},
|
||||
"type": "section"
|
||||
}
|
||||
]
|
||||
}`,
|
||||
},
|
||||
"slack_recvX/slack_testX": {
|
||||
`{
|
||||
"channel": "#test-channel",
|
||||
"username": "Integration Test",
|
||||
"attachments": [
|
||||
{
|
||||
"title": "[FIRING:1] (SlackAlert2 UID_SlackAlert2)",
|
||||
"title_link": "http:/localhost:3000/alerting/list",
|
||||
"text": "\n**Firing**\nLabels:\n - alertname = SlackAlert2\n - __alert_rule_uid__ = UID_SlackAlert2\nAnnotations:\nSource: \n\n\n\n\n",
|
||||
"fallback": "[FIRING:1] (SlackAlert2 UID_SlackAlert2)",
|
||||
"footer": "Grafana v",
|
||||
"footer_icon": "https://grafana.com/assets/img/fav32.png",
|
||||
"color": "#D63232",
|
||||
"ts": %s
|
||||
}
|
||||
],
|
||||
"blocks": [
|
||||
{
|
||||
"text": {
|
||||
"text": "<@user1><@user2>",
|
||||
"type": "mrkdwn"
|
||||
},
|
||||
"type": "section"
|
||||
}
|
||||
]
|
||||
}`,
|
||||
},
|
||||
"pagerduty_recvX/pagerduty_testX": {
|
||||
`{
|
||||
"routing_key": "pagerduty_recv/pagerduty_test",
|
||||
"dedup_key": "718643b9694d44f7f2b21458afd1b079cb403cf264e51894ff3c9745238bcced",
|
||||
"description": "[firing:1] (PagerdutyAlert UID_PagerdutyAlert)",
|
||||
"event_action": "trigger",
|
||||
"payload": {
|
||||
"summary": "Integration Test [FIRING:1] (PagerdutyAlert UID_PagerdutyAlert)",
|
||||
"source": "%s",
|
||||
"severity": "warning",
|
||||
"class": "testclass",
|
||||
"component": "Integration Test",
|
||||
"group": "testgroup",
|
||||
"custom_details": {
|
||||
"firing": "Labels:\n - alertname = PagerdutyAlert\n - __alert_rule_uid__ = UID_PagerdutyAlert\nAnnotations:\nSource: \n",
|
||||
"num_firing": "1",
|
||||
"num_resolved": "0",
|
||||
"resolved": ""
|
||||
}
|
||||
},
|
||||
"client": "Grafana",
|
||||
"client_url": "http://localhost:3000/",
|
||||
"links": [
|
||||
{
|
||||
"href": "http://localhost:3000/",
|
||||
"text": "External URL"
|
||||
}
|
||||
]
|
||||
}`,
|
||||
},
|
||||
"dingding_recv/dingding_test": {
|
||||
`{
|
||||
"link": {
|
||||
"messageUrl": "dingtalk://dingtalkclient/page/link?pc_slide=false&url=http%3A%2Flocalhost%3A3000%2Falerting%2Flist",
|
||||
"text": "\n**Firing**\nLabels:\n - alertname = DingDingAlert\n - __alert_rule_uid__ = UID_DingDingAlert\nAnnotations:\nSource: \n\n\n\n\n",
|
||||
"title": "[firing:1] (DingDingAlert UID_DingDingAlert)"
|
||||
},
|
||||
"msgtype": "link"
|
||||
}`,
|
||||
},
|
||||
"teams_recv/teams_test": {
|
||||
`{
|
||||
"@context": "http://schema.org/extensions",
|
||||
"@type": "MessageCard",
|
||||
"potentialAction": [
|
||||
{
|
||||
"@context": "http://schema.org",
|
||||
"@type": "OpenUri",
|
||||
"name": "View Rule",
|
||||
"targets": [
|
||||
{
|
||||
"os": "default",
|
||||
"uri": "http:/localhost:3000/alerting/list"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"sections": [
|
||||
{
|
||||
"text": "\n**Firing**\nLabels:\n - alertname = TeamsAlert\n - __alert_rule_uid__ = UID_TeamsAlert\nAnnotations:\nSource: \n\n\n\n\n",
|
||||
"title": "Details"
|
||||
}
|
||||
],
|
||||
"summary": "[firing:1] (TeamsAlert UID_TeamsAlert)",
|
||||
"themeColor": "#D63232",
|
||||
"title": "[firing:1] (TeamsAlert UID_TeamsAlert)"
|
||||
}`,
|
||||
},
|
||||
"webhook_recv/webhook_test": {
|
||||
`{
|
||||
"receiver": "webhook_recv",
|
||||
"status": "firing",
|
||||
"alerts": [
|
||||
{
|
||||
"status": "firing",
|
||||
"labels": {
|
||||
"__alert_rule_uid__": "UID_WebhookAlert",
|
||||
"alertname": "WebhookAlert"
|
||||
},
|
||||
"annotations": {},
|
||||
"startsAt": "%s",
|
||||
"endsAt": "0001-01-01T00:00:00Z",
|
||||
"generatorURL": "",
|
||||
"fingerprint": "929467973978d053"
|
||||
}
|
||||
],
|
||||
"groupLabels": {},
|
||||
"commonLabels": {
|
||||
"__alert_rule_uid__": "UID_WebhookAlert",
|
||||
"alertname": "WebhookAlert"
|
||||
},
|
||||
"commonAnnotations": {},
|
||||
"externalURL": "http://localhost:3000/",
|
||||
"version": "1",
|
||||
"groupKey": "{}/{alertname=\"WebhookAlert\"}:{}",
|
||||
"truncatedAlerts": 0,
|
||||
"title": "[FIRING:1] (WebhookAlert UID_WebhookAlert)",
|
||||
"state": "alerting",
|
||||
"message": "\n**Firing**\nLabels:\n - alertname = WebhookAlert\n - __alert_rule_uid__ = UID_WebhookAlert\nAnnotations:\nSource: \n\n\n\n\n"
|
||||
}`,
|
||||
},
|
||||
}
|
||||
@ -202,6 +202,11 @@ func CreateGrafDir(t *testing.T, opts ...GrafanaOpts) (string, string) {
|
||||
_, err = anonSect.NewKey("enabled", "true")
|
||||
require.NoError(t, err)
|
||||
|
||||
alertingSect, err := cfg.NewSection("alerting")
|
||||
require.NoError(t, err)
|
||||
_, err = alertingSect.NewKey("notification_timeout_seconds", "1")
|
||||
require.NoError(t, err)
|
||||
|
||||
for _, o := range opts {
|
||||
if o.EnableCSP {
|
||||
securitySect, err := cfg.NewSection("security")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user