IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Improve notification policies created during migration (#52071)

Browse Source 
* Alerting: Improve notification policies created during migration

Previously, migrated legacy alerts were connected to notification policies through
a `rule_uid` label in a 1:1 fashion. While this correctly mimicked pre-migration routing,
it didn't create a notification policy structure that is easy to view/modify. In addition,
having one policy per migrated alert is, in some ways, counter to the recommended approach of
Unified Alerting.

This change replaces `rule_uid`-based migrated notification policies with a private
label called `__contacts__`. This label stores a list of double quoted strings containing the names of
all contact points an AlertRule should route to (based on legacy notification channels). Finally,
one notification policy is created per contact point with each matching AlertRules via regex on this
`__contacts__` label.

The result is a simpler, clearer, and easier to modify notification policy structure, with the
added benefit that you can see which contact points an AlertRule is being routed to from the
AlertRule creation page.
This commit is contained in:
Matthew Jacobson 2022-10-18 00:47:39 -04:00 committed by GitHub
parent 313c88f3e1
commit 0db339d82f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 410 additions and 291 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
pkg/services/sqlstore/migrations/ualert/alert_rule.go
View File

@ -13,6 +13,13 @@ import (
"github.com/grafana/grafana/pkg/util"
)
const (
// ContactLabel is a private label created during migration and used in notification policies.
// It stores a string array of all contact point names an alert rule should send to.
// It was created as a means to simplify post-migration notification policies.
ContactLabel = "__contacts__"
)
type alertRule struct {
ID int64 `xorm:"pk autoincr 'id'"`
OrgID int64 `xorm:"org_id"`
@ -30,7 +37,7 @@ type alertRule struct {
For duration
Updated time.Time
Annotations map[string]string
Labels map[string]string // (Labels are not Created in the migration)
Labels map[string]string
}
type alertRuleVersion struct {
@ -136,7 +143,7 @@ func (m *migration) makeAlertRule(cond condition, da dashAlert, folderUID string
}
// Label for routing and silences.
n, v := getLabelForRouteMatching(ar.UID)
n, v := getLabelForSilenceMatching(ar.UID)
ar.Labels[n] = v
if err := m.addSilence(da, ar); err != nil {
@ -290,3 +297,20 @@ func normalizeRuleName(daName string) string {
return daName
}
func extractChannelIDs(d dashAlert) (channelUids []uidOrID) {
// Extracting channel UID/ID.
for _, ui := range d.ParsedSettings.Notifications {
if ui.UID != "" {
channelUids = append(channelUids, ui.UID)
continue
}
// In certain circumstances, id is used instead of uid.
// We add this if there was no uid.
if ui.ID > 0 {
channelUids = append(channelUids, ui.ID)
}
}
return channelUids
}

167
pkg/services/sqlstore/migrations/ualert/channel.go
View File

@ -1,10 +1,14 @@
package ualert
import (
"crypto/md5"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"regexp"
"sort"
"strings"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/common/model"
@ -36,7 +40,7 @@ type defaultChannelsPerOrg map[int64][]*notificationChannel
type uidOrID interface{}
// setupAlertmanagerConfigs creates Alertmanager configs with migrated receivers and routes.
func (m *migration) setupAlertmanagerConfigs(rulesPerOrg map[int64]map[string]dashAlert) (amConfigsPerOrg, error) {
func (m *migration) setupAlertmanagerConfigs(rulesPerOrg map[int64]map[*alertRule][]uidOrID) (amConfigsPerOrg, error) {
// allChannels: channelUID -> channelConfig
allChannelsPerOrg, defaultChannelsPerOrg, err := m.getNotificationChannelMap()
if err != nil {
@ -83,17 +87,21 @@ func (m *migration) setupAlertmanagerConfigs(rulesPerOrg map[int64]map[string]da
amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, defaultReceiver)
}
// Create routes
if rules, ok := rulesPerOrg[orgID]; ok {
for ruleUid, da := range rules {
route, err := m.createRouteForAlert(ruleUid, da, receiversMap, defaultReceivers)
if err != nil {
return nil, fmt.Errorf("failed to create route for alert %s in orgId %d: %w", da.Name, orgID, err)
}
for _, recv := range receivers {
route, err := createRoute(recv)
if err != nil {
return nil, fmt.Errorf("failed to create route for receiver %s in orgId %d: %w", recv.Name, orgID, err)
}
if route != nil {
amConfigPerOrg[da.OrgId].AlertmanagerConfig.Route.Routes = append(amConfigPerOrg[da.OrgId].AlertmanagerConfig.Route.Routes, route)
}
amConfigPerOrg[orgID].AlertmanagerConfig.Route.Routes = append(amConfigPerOrg[orgID].AlertmanagerConfig.Route.Routes, route)
}
for ar, channelUids := range rulesPerOrg[orgID] {
filteredReceiverNames := m.filterReceiversForAlert(ar.Title, channelUids, receiversMap, defaultReceivers)
if len(filteredReceiverNames) != 0 {
// Only create a contact label if there are specific receivers, otherwise it defaults to the root-level route.
ar.Labels[ContactLabel] = contactListToString(filteredReceiverNames)
}
}
@ -107,6 +115,22 @@ func (m *migration) setupAlertmanagerConfigs(rulesPerOrg map[int64]map[string]da
return amConfigPerOrg, nil
}
// contactListToString creates a sorted string representation of a given map (set) of receiver names. Each name will be comma-separated and double-quoted. Names should not contain double quotes.
func contactListToString(m map[string]interface{}) string {
keys := make([]string, 0, len(m))
for k := range m {
keys = append(keys, quote(k))
}
sort.Strings(keys)
return strings.Join(keys, ",")
}
// quote will surround the given string in double quotes.
func quote(s string) string {
return `"` + s + `"`
}
// getNotificationChannelMap returns a map of all channelUIDs to channel config as well as a separate map for just those channels that are default.
// For any given Organization, all channels in defaultChannelsPerOrg should also exist in channelsPerOrg.
func (m *migration) getNotificationChannelMap() (channelsPerOrg, defaultChannelsPerOrg, error) {
@ -177,14 +201,27 @@ func (m *migration) createNotifier(c *notificationChannel) (*PostableGrafanaRece
func (m *migration) createReceivers(allChannels []*notificationChannel) (map[uidOrID]*PostableApiReceiver, []*PostableApiReceiver, error) {
var receivers []*PostableApiReceiver
receiversMap := make(map[uidOrID]*PostableApiReceiver)
set := make(map[string]struct{}) // Used to deduplicate sanitized names.
for _, c := range allChannels {
notifier, err := m.createNotifier(c)
if err != nil {
return nil, nil, err
}
// We remove double quotes because this character will be used as the separator in the ContactLabel. To prevent partial matches in the Route Matcher we choose to sanitize them early on instead of complicating the Matcher regex.
sanitizedName := strings.ReplaceAll(c.Name, `"`, `_`)
// There can be name collisions after we sanitize. We check for this and attempt to make the name unique again using a short hash of the original name.
if _, ok := set[sanitizedName]; ok {
sanitizedName = sanitizedName + fmt.Sprintf("_%.3x", md5.Sum([]byte(c.Name)))
m.mg.Logger.Warn("alert contains duplicate contact name after sanitization, appending unique suffix", "type", c.Type, "name", c.Name, "new_name", sanitizedName, "uid", c.Uid)
}
notifier.Name = sanitizedName
set[sanitizedName] = struct{}{}
recv := &PostableApiReceiver{
Name: c.Name, // Channel name is unique within an Org.
Name: sanitizedName, // Channel name is unique within an Org.
GrafanaManagedReceivers: []*PostableGrafanaReceiver{notifier},
}
@ -239,64 +276,28 @@ func (m *migration) createDefaultRouteAndReceiver(defaultChannels []*notificatio
return defaultReceiver, defaultRoute, nil
}
// Wrapper to select receivers for given alert rules based on associated notification channels and then create the migrated route.
func (m *migration) createRouteForAlert(ruleUID string, da dashAlert, receivers map[uidOrID]*PostableApiReceiver, defaultReceivers map[string]struct{}) (*Route, error) {
// Create route(s) for alert
filteredReceiverNames := m.filterReceiversForAlert(da, receivers, defaultReceivers)
// Create one route per contact point, matching based on ContactLabel.
func createRoute(recv *PostableApiReceiver) (*Route, error) {
// We create a regex matcher so that each alert rule need only have a single ContactLabel entry for all contact points it sends to.
// For example, if an alert needs to send to contact1 and contact2 it will have ContactLabel=`"contact1","contact2"` and will match both routes looking
// for `.*"contact1".*` and `.*"contact2".*`.
if len(filteredReceiverNames) != 0 {
// Only create a route if there are specific receivers, otherwise it defaults to the root-level route.
route, err := createRoute(ruleUID, filteredReceiverNames)
if err != nil {
return nil, err
}
return route, nil
}
return nil, nil
}
// Create route(s) for the given alert ruleUID and receivers.
// If the alert had a single channel, it will now have a single route/policy. If the alert had multiple channels, it will now have multiple nested routes/policies.
func createRoute(ruleUID string, filteredReceiverNames map[string]interface{}) (*Route, error) {
n, v := getLabelForRouteMatching(ruleUID)
mat, err := labels.NewMatcher(labels.MatchEqual, n, v)
// We quote and escape here to ensure the regex will correctly match the ContactLabel on the alerts.
name := fmt.Sprintf(`.*%s.*`, regexp.QuoteMeta(quote(recv.Name)))
mat, err := labels.NewMatcher(labels.MatchRegexp, ContactLabel, name)
if err != nil {
return nil, err
}
var route *Route
if len(filteredReceiverNames) == 1 {
for name := range filteredReceiverNames {
route = &Route{
Receiver: name,
Matchers: Matchers{mat},
}
}
} else {
nestedRoutes := []*Route{}
for name := range filteredReceiverNames {
r := &Route{
Receiver: name,
Matchers: Matchers{mat},
Continue: true,
}
nestedRoutes = append(nestedRoutes, r)
}
route = &Route{
Matchers: Matchers{mat},
Routes: nestedRoutes,
}
}
return route, nil
return &Route{
Receiver: recv.Name,
ObjectMatchers: ObjectMatchers{mat},
Continue: true, // We continue so that each sibling contact point route can separately match.
}, nil
}
// Filter receivers to select those that were associated to the given rule as channels.
func (m *migration) filterReceiversForAlert(da dashAlert, receivers map[uidOrID]*PostableApiReceiver, defaultReceivers map[string]struct{}) map[string]interface{} {
channelIDs := extractChannelIDs(da)
func (m *migration) filterReceiversForAlert(name string, channelIDs []uidOrID, receivers map[uidOrID]*PostableApiReceiver, defaultReceivers map[string]struct{}) map[string]interface{} {
if len(channelIDs) == 0 {
// If there are no channels associated, we use the default route.
return nil
@ -309,7 +310,7 @@ func (m *migration) filterReceiversForAlert(da dashAlert, receivers map[uidOrID]
if ok {
filteredReceiverNames[recv.Name] = struct{}{} // Deduplicate on contact point name.
} else {
m.mg.Logger.Warn("alert linked to obsolete notification channel, ignoring", "alert", da.Name, "uid", uidOrId)
m.mg.Logger.Warn("alert linked to obsolete notification channel, ignoring", "alert", name, "uid", uidOrId)
}
}
@ -403,27 +404,6 @@ func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json,
return cloneSettings, newSecureSettings, nil
}
func getLabelForRouteMatching(ruleUID string) (string, string) {
return "rule_uid", ruleUID
}
func extractChannelIDs(d dashAlert) (channelUids []uidOrID) {
// Extracting channel UID/ID.
for _, ui := range d.ParsedSettings.Notifications {
if ui.UID != "" {
channelUids = append(channelUids, ui.UID)
continue
}
// In certain circumstances, id is used instead of uid.
// We add this if there was no uid.
if ui.ID > 0 {
channelUids = append(channelUids, ui.ID)
}
}
return channelUids
}
// Below is a snapshot of all the config and supporting functions imported
// to avoid vendoring those packages.
@ -441,22 +421,23 @@ type PostableApiAlertingConfig struct {
}
type Route struct {
Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty"`
Matchers Matchers `yaml:"matchers,omitempty" json:"matchers,omitempty"`
Routes []*Route `yaml:"routes,omitempty" json:"routes,omitempty"`
Continue bool `yaml:"continue,omitempty" json:"continue,omitempty"`
GroupByStr []string `yaml:"group_by,omitempty" json:"group_by,omitempty"`
Receiver string `yaml:"receiver,omitempty" json:"receiver,omitempty"`
ObjectMatchers ObjectMatchers `yaml:"object_matchers,omitempty" json:"object_matchers,omitempty"`
Routes []*Route `yaml:"routes,omitempty" json:"routes,omitempty"`
Continue bool `yaml:"continue,omitempty" json:"continue,omitempty"`
GroupByStr []string `yaml:"group_by,omitempty" json:"group_by,omitempty"`
}
type Matchers labels.Matchers
type ObjectMatchers labels.Matchers
func (m Matchers) MarshalJSON() ([]byte, error) {
// MarshalJSON implements the json.Marshaler interface for Matchers. Vendored from definitions.ObjectMatchers.
func (m ObjectMatchers) MarshalJSON() ([]byte, error) {
if len(m) == 0 {
return nil, nil
}
result := make([]string, len(m))
result := make([][3]string, len(m))
for i, matcher := range m {
result[i] = matcher.String()
result[i] = [3]string{matcher.Name, matcher.Type.String(), matcher.Value}
}
return json.Marshal(result)
}

177
pkg/services/sqlstore/migrations/ualert/channel_test.go
View File

@ -3,6 +3,9 @@ package ualert
import (
"testing"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/prometheus/alertmanager/pkg/labels"
"github.com/prometheus/common/model"
"github.com/stretchr/testify/require"
@ -13,18 +16,14 @@ import (
func TestFilterReceiversForAlert(t *testing.T) {
tc := []struct {
name string
da dashAlert
channelIds []uidOrID
receivers map[uidOrID]*PostableApiReceiver
defaultReceivers map[string]struct{}
expected map[string]interface{}
}{
{
name: "when an alert has multiple channels, each should filter for the correct receiver",
da: dashAlert{
ParsedSettings: &dashAlertSettings{
Notifications: []dashAlertNot{{UID: "uid1"}, {UID: "uid2"}},
},
},
name: "when an alert has multiple channels, each should filter for the correct receiver",
channelIds: []uidOrID{"uid1", "uid2"},
receivers: map[uidOrID]*PostableApiReceiver{
"uid1": {
Name: "recv1",
@ -46,12 +45,8 @@ func TestFilterReceiversForAlert(t *testing.T) {
},
},
{
name: "when default receivers exist, they should be added to an alert's filtered receivers",
da: dashAlert{
ParsedSettings: &dashAlertSettings{
Notifications: []dashAlertNot{{UID: "uid1"}},
},
},
name: "when default receivers exist, they should be added to an alert's filtered receivers",
channelIds: []uidOrID{"uid1"},
receivers: map[uidOrID]*PostableApiReceiver{
"uid1": {
Name: "recv1",
@ -75,12 +70,8 @@ func TestFilterReceiversForAlert(t *testing.T) {
},
},
{
name: "when an alert has a channels associated by ID instead of UID, it should be included",
da: dashAlert{
ParsedSettings: &dashAlertSettings{
Notifications: []dashAlertNot{{ID: int64(42)}},
},
},
name: "when an alert has a channels associated by ID instead of UID, it should be included",
channelIds: []uidOrID{int64(42)},
receivers: map[uidOrID]*PostableApiReceiver{
int64(42): {
Name: "recv1",
@ -93,12 +84,8 @@ func TestFilterReceiversForAlert(t *testing.T) {
},
},
{
name: "when an alert's receivers are covered by the defaults, return nil to use default receiver downstream",
da: dashAlert{
ParsedSettings: &dashAlertSettings{
Notifications: []dashAlertNot{{UID: "uid1"}},
},
},
name: "when an alert's receivers are covered by the defaults, return nil to use default receiver downstream",
channelIds: []uidOrID{"uid1"},
receivers: map[uidOrID]*PostableApiReceiver{
"uid1": {
Name: "recv1",
@ -124,7 +111,7 @@ func TestFilterReceiversForAlert(t *testing.T) {
for _, tt := range tc {
t.Run(tt.name, func(t *testing.T) {
m := newTestMigration(t)
res := m.filterReceiversForAlert(tt.da, tt.receivers, tt.defaultReceivers)
res := m.filterReceiversForAlert("", tt.channelIds, tt.receivers, tt.defaultReceivers)
require.Equal(t, tt.expected, res)
})
@ -133,77 +120,57 @@ func TestFilterReceiversForAlert(t *testing.T) {
func TestCreateRoute(t *testing.T) {
tc := []struct {
name string
ruleUID string
filteredReceiverNames map[string]interface{}
expected *Route
expErr error
name string
recv *PostableApiReceiver
expected *Route
}{
{
name: "when a single receiver is passed in, the route should be simple and not nested",
ruleUID: "r_uid1",
filteredReceiverNames: map[string]interface{}{
"recv1": struct{}{},
name: "when a receiver is passed in, the route should regex match based on quoted name with continue=true",
recv: &PostableApiReceiver{
Name: "recv1",
},
expected: &Route{
Receiver: "recv1",
Matchers: Matchers{{Type: 0, Name: "rule_uid", Value: "r_uid1"}},
Routes: nil,
Continue: false,
GroupByStr: nil,
Receiver: "recv1",
ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"recv1".*`}},
Routes: nil,
Continue: true,
GroupByStr: nil,
},
},
{
name: "when multiple receivers are passed in, the route should be nested with continue=true",
ruleUID: "r_uid1",
filteredReceiverNames: map[string]interface{}{
"recv1": struct{}{},
"recv2": struct{}{},
name: "notification channel should be escaped for regex in the matcher",
recv: &PostableApiReceiver{
Name: `. ^ $ * + - ? ( ) [ ] { } \ |`,
},
expected: &Route{
Receiver: "",
Matchers: Matchers{{Type: 0, Name: "rule_uid", Value: "r_uid1"}},
Routes: []*Route{
{
Receiver: "recv1",
Matchers: Matchers{{Type: 0, Name: "rule_uid", Value: "r_uid1"}},
Routes: nil,
Continue: true,
GroupByStr: nil,
},
{
Receiver: "recv2",
Matchers: Matchers{{Type: 0, Name: "rule_uid", Value: "r_uid1"}},
Routes: nil,
Continue: true,
GroupByStr: nil,
},
},
Continue: false,
GroupByStr: nil,
Receiver: `. ^ $ * + - ? ( ) [ ] { } \ |`,
ObjectMatchers: ObjectMatchers{{Type: 2, Name: ContactLabel, Value: `.*"\. \^ \$ \* \+ - \? \( \) \[ \] \{ \} \\ \|".*`}},
Routes: nil,
Continue: true,
GroupByStr: nil,
},
},
}
for _, tt := range tc {
t.Run(tt.name, func(t *testing.T) {
res, err := createRoute(tt.ruleUID, tt.filteredReceiverNames)
if tt.expErr != nil {
require.Error(t, err)
require.EqualError(t, err, tt.expErr.Error())
return
}
res, err := createRoute(tt.recv)
require.NoError(t, err)
// Compare route slice separately since order is not guaranteed
expRoutes := tt.expected.Routes
tt.expected.Routes = nil
actRoutes := res.Routes
res.Routes = nil
// Order of nested routes is not guaranteed.
cOpt := []cmp.Option{
cmpopts.SortSlices(func(a, b *Route) bool {
if a.Receiver != b.Receiver {
return a.Receiver < b.Receiver
}
return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value
}),
cmpopts.IgnoreUnexported(Route{}, labels.Matcher{}),
}
require.Equal(t, tt.expected, res)
require.ElementsMatch(t, expRoutes, actRoutes)
if !cmp.Equal(tt.expected, res, cOpt...) {
t.Errorf("Unexpected Route: %v", cmp.Diff(tt.expected, res, cOpt...))
}
})
}
}
@ -254,6 +221,58 @@ func TestCreateReceivers(t *testing.T) {
},
},
},
{
name: "when given notification channel contains double quote sanitize with underscore",
allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name\"1")},
expRecvMap: map[uidOrID]*PostableApiReceiver{
"uid1": {
Name: "name_1",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}},
},
int64(1): {
Name: "name_1",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}},
},
},
expRecv: []*PostableApiReceiver{
{
Name: "name_1",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}},
},
},
},
{
name: "when given notification channels collide after sanitization add short hash to end",
allChannels: []*notificationChannel{createNotChannel(t, "uid1", int64(1), "name\"1"), createNotChannel(t, "uid2", int64(2), "name_1")},
expRecvMap: map[uidOrID]*PostableApiReceiver{
"uid1": {
Name: "name_1",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}},
},
"uid2": {
Name: "name_1_dba13d",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}},
},
int64(1): {
Name: "name_1",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}},
},
int64(2): {
Name: "name_1_dba13d",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}},
},
},
expRecv: []*PostableApiReceiver{
{
Name: "name_1",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1"}},
},
{
Name: "name_1_dba13d",
GrafanaManagedReceivers: []*PostableGrafanaReceiver{{Name: "name_1_dba13d"}},
},
},
},
}
for _, tt := range tc {

218
pkg/services/sqlstore/migrations/ualert/migration_test.go
View File

@ -119,15 +119,11 @@ func TestAddDashAlertMigration(t *testing.T) {
}
}
// TestDashAlertMigration tests the execution of the main DashAlertMigration.
func TestDashAlertMigration(t *testing.T) {
// TestAMConfigMigration tests the execution of the main DashAlertMigration specifically for migrations of channels and routes.
func TestAMConfigMigration(t *testing.T) {
// Run initial migration to have a working DB.
x := setupTestDB(t)
emailSettings := `{"addresses": "test"}`
slackSettings := `{"recipient": "test", "token": "test"}`
opsgenieSettings := `{"apiKey": "test"}`
tc := []struct {
name string
legacyChannels []*models.AlertNotification
@ -161,12 +157,9 @@ func TestDashAlertMigration(t *testing.T) {
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", Matchers: createAlertNameMatchers("alert1")}, // These Matchers are temporary and will be replaced below with generated rule_uid.
{Matchers: createAlertNameMatchers("alert2"), Routes: []*ualert.Route{
{Receiver: "notifier2", Matchers: createAlertNameMatchers("alert2"), Continue: true},
{Receiver: "notifier3", Matchers: createAlertNameMatchers("alert2"), Continue: true},
}},
{Receiver: "notifier3", Matchers: createAlertNameMatchers("alert3")},
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier3", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
@ -183,15 +176,9 @@ func TestDashAlertMigration(t *testing.T) {
Receiver: "notifier6",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Matchers: createAlertNameMatchers("alert4"), Routes: []*ualert.Route{
{Receiver: "notifier4", Matchers: createAlertNameMatchers("alert4"), Continue: true},
{Receiver: "notifier6", Matchers: createAlertNameMatchers("alert4"), Continue: true},
}},
{Matchers: createAlertNameMatchers("alert5"), Routes: []*ualert.Route{
{Receiver: "notifier4", Matchers: createAlertNameMatchers("alert5"), Continue: true},
{Receiver: "notifier5", Matchers: createAlertNameMatchers("alert5"), Continue: true},
{Receiver: "notifier6", Matchers: createAlertNameMatchers("alert5"), Continue: true},
}},
{Receiver: "notifier4", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier4".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier5", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier5".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier6", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier6".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
@ -215,6 +202,9 @@ func TestDashAlertMigration(t *testing.T) {
Route: &ualert.Route{
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
{Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}},
@ -236,6 +226,9 @@ func TestDashAlertMigration(t *testing.T) {
Route: &ualert.Route{
Receiver: "notifier1",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
{Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}},
@ -257,6 +250,10 @@ func TestDashAlertMigration(t *testing.T) {
Route: &ualert.Route{
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
{Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}},
@ -281,6 +278,11 @@ func TestDashAlertMigration(t *testing.T) {
Route: &ualert.Route{
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier3", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier3".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
{Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}},
@ -307,6 +309,10 @@ func TestDashAlertMigration(t *testing.T) {
Route: &ualert.Route{
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
{Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}},
@ -334,11 +340,8 @@ func TestDashAlertMigration(t *testing.T) {
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", Matchers: createAlertNameMatchers("alert1")},
{Matchers: createAlertNameMatchers("alert2"), Routes: []*ualert.Route{
{Receiver: "notifier1", Matchers: createAlertNameMatchers("alert2"), Continue: true},
{Receiver: "notifier2", Matchers: createAlertNameMatchers("alert2"), Continue: true},
}},
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
{Receiver: "notifier2", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier2".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
@ -362,6 +365,9 @@ func TestDashAlertMigration(t *testing.T) {
Route: &ualert.Route{
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
{Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}},
@ -385,6 +391,9 @@ func TestDashAlertMigration(t *testing.T) {
Route: &ualert.Route{
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
{Name: "notifier1", GrafanaManagedReceivers: []*ualert.PostableGrafanaReceiver{{Name: "notifier1", Type: "email"}}},
@ -410,7 +419,7 @@ func TestDashAlertMigration(t *testing.T) {
Receiver: "autogen-contact-point-default",
GroupByStr: []string{ngModels.FolderTitleLabel, model.AlertNameLabel},
Routes: []*ualert.Route{
{Receiver: "notifier1", Matchers: createAlertNameMatchers("alert1")},
{Receiver: "notifier1", ObjectMatchers: ualert.ObjectMatchers{{Type: 2, Name: ualert.ContactLabel, Value: `.*"notifier1".*`}}, Routes: nil, Continue: true},
},
},
Receivers: []*ualert.PostableApiReceiver{
@ -427,16 +436,7 @@ func TestDashAlertMigration(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
defer teardown(t, x)
setupLegacyAlertsTables(t, x, tt.legacyChannels, tt.alerts)
_, errDeleteMig := x.Exec("DELETE FROM migration_log WHERE migration_id = ?", ualert.MigTitle)
require.NoError(t, errDeleteMig)
alertMigrator := migrator.NewMigrator(x, &setting.Cfg{})
alertMigrator.AddMigration(ualert.RmMigTitle, &ualert.RmMigration{})
ualert.AddDashAlertMigration(alertMigrator)
errRunningMig := alertMigrator.Start(false, 0)
require.NoError(t, errRunningMig)
runDashAlertMigrationTestRun(t, x)
for orgId := range tt.expected {
amConfig := getAlertmanagerConfig(t, x, orgId)
@ -451,17 +451,13 @@ func TestDashAlertMigration(t *testing.T) {
t.Errorf("Unexpected Receivers: %v", cmp.Diff(tt.expected[orgId].AlertmanagerConfig.Receivers, amConfig.AlertmanagerConfig.Receivers, cOpt...))
}
// Since routes and alerts are connecting solely by the Matchers on rule_uid, which is created at runtime we need to do some prep-work to populate the expected Matchers.
alertUids := getAlertNameToUidMap(t, x, orgId)
replaceAlertNameMatcherWithRuleUid(t, tt.expected[orgId].AlertmanagerConfig.Route.Routes, alertUids)
// Order of nested routes is not guaranteed.
// Order of routes is not guaranteed.
cOpt = []cmp.Option{
cmpopts.SortSlices(func(a, b *ualert.Route) bool {
if a.Receiver != b.Receiver {
return a.Receiver < b.Receiver
}
return a.Matchers[0].Value < b.Matchers[0].Value
return a.ObjectMatchers[0].Value < b.ObjectMatchers[0].Value
}),
cmpopts.IgnoreUnexported(ualert.Route{}, labels.Matcher{}),
}
@ -473,6 +469,87 @@ func TestDashAlertMigration(t *testing.T) {
}
}
// TestDashAlertMigration tests the execution of the main DashAlertMigration specifically for migrations of alerts.
func TestDashAlertMigration(t *testing.T) {
// Run initial migration to have a working DB.
x := setupTestDB(t)
t.Run("when DashAlertMigration create ContactLabel on migrated AlertRules", func(t *testing.T) {
defer teardown(t, x)
legacyChannels := []*models.AlertNotification{
createAlertNotification(t, int64(1), "notifier1", "email", emailSettings, false),
createAlertNotification(t, int64(1), "notifier2", "slack", slackSettings, false),
createAlertNotification(t, int64(1), "notifier3", "opsgenie", opsgenieSettings, false),
createAlertNotification(t, int64(2), "notifier4", "email", emailSettings, false),
createAlertNotification(t, int64(2), "notifier5", "slack", slackSettings, false),
createAlertNotification(t, int64(2), "notifier6", "opsgenie", opsgenieSettings, true), // default
}
alerts := []*models.Alert{
createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notifier1"}),
createAlert(t, int64(1), int64(1), int64(2), "alert2", []string{"notifier2", "notifier3"}),
createAlert(t, int64(1), int64(2), int64(3), "alert3", []string{"notifier3"}),
createAlert(t, int64(2), int64(3), int64(1), "alert4", []string{"notifier4"}),
createAlert(t, int64(2), int64(3), int64(2), "alert5", []string{"notifier4", "notifier5", "notifier6"}),
createAlert(t, int64(2), int64(4), int64(3), "alert6", []string{}),
}
expected := map[int64]map[string]*ngModels.AlertRule{
int64(1): {
"alert1": {Labels: map[string]string{ualert.ContactLabel: `"notifier1"`}},
"alert2": {Labels: map[string]string{ualert.ContactLabel: `"notifier2","notifier3"`}},
"alert3": {Labels: map[string]string{ualert.ContactLabel: `"notifier3"`}},
},
int64(2): {
"alert4": {Labels: map[string]string{ualert.ContactLabel: `"notifier4","notifier6"`}},
"alert5": {Labels: map[string]string{ualert.ContactLabel: `"notifier4","notifier5","notifier6"`}},
"alert6": {Labels: map[string]string{}},
},
}
setupLegacyAlertsTables(t, x, legacyChannels, alerts)
runDashAlertMigrationTestRun(t, x)
for orgId := range expected {
rules := getAlertRules(t, x, orgId)
expectedRulesMap := expected[orgId]
require.Len(t, rules, len(expectedRulesMap))
for _, r := range rules {
require.Equal(t, expectedRulesMap[r.Title].Labels[ualert.ContactLabel], r.Labels[ualert.ContactLabel])
}
}
})
t.Run("when DashAlertMigration create ContactLabel with sanitized name if name contains double quote", func(t *testing.T) {
defer teardown(t, x)
legacyChannels := []*models.AlertNotification{
createAlertNotification(t, int64(1), "notif\"ier1", "email", emailSettings, false),
}
alerts := []*models.Alert{
createAlert(t, int64(1), int64(1), int64(1), "alert1", []string{"notif\"ier1"}),
}
expected := map[int64]map[string]*ngModels.AlertRule{
int64(1): {
"alert1": {Labels: map[string]string{ualert.ContactLabel: `"notif_ier1"`}},
},
}
setupLegacyAlertsTables(t, x, legacyChannels, alerts)
runDashAlertMigrationTestRun(t, x)
for orgId := range expected {
rules := getAlertRules(t, x, orgId)
expectedRulesMap := expected[orgId]
require.Len(t, rules, len(expectedRulesMap))
for _, r := range rules {
require.Equal(t, expectedRulesMap[r.Title].Labels[ualert.ContactLabel], r.Labels[ualert.ContactLabel])
}
}
})
}
const (
emailSettings = `{"addresses": "test"}`
slackSettings = `{"recipient": "test", "token": "test"}`
opsgenieSettings = `{"apiKey": "test"}`
)
// setupTestDB prepares the sqlite database and runs OSS migrations to initialize the schemas.
func setupTestDB(t *testing.T) *xorm.Engine {
t.Helper()
@ -607,6 +684,19 @@ func teardown(t *testing.T, x *xorm.Engine) {
require.NoError(t, err)
}
// setupDashAlertMigrationTestRun runs DashAlertMigration for a new test run.
func runDashAlertMigrationTestRun(t *testing.T, x *xorm.Engine) {
_, errDeleteMig := x.Exec("DELETE FROM migration_log WHERE migration_id = ?", ualert.MigTitle)
require.NoError(t, errDeleteMig)
alertMigrator := migrator.NewMigrator(x, &setting.Cfg{})
alertMigrator.AddMigration(ualert.RmMigTitle, &ualert.RmMigration{})
ualert.AddDashAlertMigration(alertMigrator)
errRunningMig := alertMigrator.Start(false, 0)
require.NoError(t, errRunningMig)
}
// setupLegacyAlertsTables inserts data into the legacy alerting tables that is needed for testing the migration.
func setupLegacyAlertsTables(t *testing.T, x *xorm.Engine, legacyChannels []*models.AlertNotification, alerts []*models.Alert) {
t.Helper()
@ -663,47 +753,15 @@ func getAlertmanagerConfig(t *testing.T, x *xorm.Engine, orgId int64) *ualert.Po
return &config
}
// getAlertNameToUidMap fetches alert_rules from database to create map of alert.Name -> alert.Uid. This is needed as alert Uid is created during migration and is used to match routes to alerts.
func getAlertNameToUidMap(t *testing.T, x *xorm.Engine, orgId int64) map[string]string {
t.Helper()
alerts := []struct {
Title string
Uid string
}{}
err := x.Table("alert_rule").Where("org_id = ?", orgId).Find(&alerts)
// getAlertmanagerConfig retreives the Alertmanager Config from the database for a given orgId.
func getAlertRules(t *testing.T, x *xorm.Engine, orgId int64) []*ngModels.AlertRule {
rules := make([]*ngModels.AlertRule, 0)
err := x.Table("alert_rule").Where("org_id = ?", orgId).Find(&rules)
require.NoError(t, err)
res := make(map[string]string)
for _, alert := range alerts {
res[alert.Title] = alert.Uid
}
return res
}
// replaceAlertNameMatcherWithRuleUid replaces the stub matchers based on alert_name with the rule_uid's generated during migration.
func replaceAlertNameMatcherWithRuleUid(t *testing.T, rts []*ualert.Route, alertUids map[string]string) {
for _, rt := range rts {
if len(rt.Matchers) > 0 {
// Replace alert name matcher with generated rule_uid matcher
for _, m := range rt.Matchers {
if m.Name == "alert_name" {
m.Name = "rule_uid"
m.Value = alertUids[m.Value]
}
}
}
// Recurse for nested routes.
replaceAlertNameMatcherWithRuleUid(t, rt.Routes, alertUids)
}
return rules
}
func boolPointer(b bool) *bool {
return &b
}
// createAlertNameMatchers creates a temporary alert_name Matchers that will be replaced during runtime with the generated rule_uid.
func createAlertNameMatchers(alertName string) ualert.Matchers {
matcher, _ := labels.NewMatcher(labels.MatchEqual, "alert_name", alertName)
return ualert.Matchers(labels.Matchers{matcher})
}

6
pkg/services/sqlstore/migrations/ualert/silences.go
View File

@ -36,7 +36,7 @@ func (m *migration) addSilence(da dashAlert, rule *alertRule) error {
return errors.New("failed to create uuid for silence")
}
n, v := getLabelForRouteMatching(rule.UID)
n, v := getLabelForSilenceMatching(rule.UID)
s := &pb.MeshSilence{
Silence: &pb.Silence{
Id: uid.String(),
@ -211,3 +211,7 @@ func openReplace(filename string) (*replaceFile, error) {
}
return rf, nil
}
func getLabelForSilenceMatching(ruleUID string) (string, string) {
return "rule_uid", ruleUID
}

74
pkg/services/sqlstore/migrations/ualert/ualert.go
View File

@ -260,8 +260,8 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
// cache for the general folders
generalFolderCache := make(map[int64]*dashboard)
// Store of newly created rules to later create routes
rulesPerOrg := make(map[int64]map[string]dashAlert)
// Per org map of newly created rules to which notification channels it should send to.
rulesPerOrg := make(map[int64]map[*alertRule][]uidOrID)
for _, da := range dashAlerts {
newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap)
@ -365,41 +365,16 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
}
if _, ok := rulesPerOrg[rule.OrgID]; !ok {
rulesPerOrg[rule.OrgID] = make(map[string]dashAlert)
rulesPerOrg[rule.OrgID] = make(map[*alertRule][]uidOrID)
}
if _, ok := rulesPerOrg[rule.OrgID][rule.UID]; !ok {
rulesPerOrg[rule.OrgID][rule.UID] = da
if _, ok := rulesPerOrg[rule.OrgID][rule]; !ok {
rulesPerOrg[rule.OrgID][rule] = extractChannelIDs(da)
} else {
return MigrationError{
Err: fmt.Errorf("duplicate generated rule UID"),
AlertId: da.Id,
}
}
if strings.HasPrefix(mg.Dialect.DriverName(), migrator.Postgres) {
err = mg.InTransaction(func(sess *xorm.Session) error {
_, err = sess.Insert(rule)
return err
})
} else {
_, err = m.sess.Insert(rule)
}
if err != nil {
// TODO better error handling, if constraint
rule.Title += fmt.Sprintf(" %v", rule.UID)
rule.RuleGroup += fmt.Sprintf(" %v", rule.UID)
_, err = m.sess.Insert(rule)
if err != nil {
return err
}
}
// create entry in alert_rule_version
_, err = m.sess.Insert(rule.makeVersion())
if err != nil {
return err
}
}
for orgID := range rulesPerOrg {
@ -412,6 +387,12 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
if err != nil {
return err
}
err = m.insertRules(mg, rulesPerOrg)
if err != nil {
return err
}
for orgID, amConfig := range amConfigPerOrg {
if err := m.writeAlertmanagerConfig(orgID, amConfig); err != nil {
return err
@ -421,6 +402,39 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
return nil
}
func (m *migration) insertRules(mg *migrator.Migrator, rulesPerOrg map[int64]map[*alertRule][]uidOrID) error {
for _, rules := range rulesPerOrg {
for rule := range rules {
var err error
if strings.HasPrefix(mg.Dialect.DriverName(), migrator.Postgres) {
err = mg.InTransaction(func(sess *xorm.Session) error {
_, err := sess.Insert(rule)
return err
})
} else {
_, err = m.sess.Insert(rule)
}
if err != nil {
// TODO better error handling, if constraint
rule.Title += fmt.Sprintf(" %v", rule.UID)
rule.RuleGroup += fmt.Sprintf(" %v", rule.UID)
_, err = m.sess.Insert(rule)
if err != nil {
return err
}
}
// create entry in alert_rule_version
_, err = m.sess.Insert(rule.makeVersion())
if err != nil {
return err
}
}
}
return nil
}
func (m *migration) writeAlertmanagerConfig(orgID int64, amConfig *PostableUserConfig) error {
rawAmConfig, err := json.Marshal(amConfig)
if err != nil {

31
pkg/services/sqlstore/migrations/ualert/ualert_test.go
View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"sort"
"strings"
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
@ -19,17 +20,35 @@ var ClearMigrationEntryTitle = clearMigrationEntryTitle
type RmMigration = rmMigration
func (m *Matchers) UnmarshalJSON(data []byte) error {
var lines []string
if err := json.Unmarshal(data, &lines); err != nil {
// UnmarshalJSON implements the json.Unmarshaler interface for Matchers. Vendored from definitions.ObjectMatchers.
func (m *ObjectMatchers) UnmarshalJSON(data []byte) error {
var rawMatchers [][3]string
if err := json.Unmarshal(data, &rawMatchers); err != nil {
return err
}
for _, line := range lines {
pm, err := labels.ParseMatchers(line)
for _, rawMatcher := range rawMatchers {
var matchType labels.MatchType
switch rawMatcher[1] {
case "=":
matchType = labels.MatchEqual
case "!=":
matchType = labels.MatchNotEqual
case "=~":
matchType = labels.MatchRegexp
case "!~":
matchType = labels.MatchNotRegexp
default:
return fmt.Errorf("unsupported match type %q in matcher", rawMatcher[1])
}
rawMatcher[2] = strings.TrimPrefix(rawMatcher[2], "\"")
rawMatcher[2] = strings.TrimSuffix(rawMatcher[2], "\"")
matcher, err := labels.NewMatcher(matchType, rawMatcher[0], rawMatcher[2])
if err != nil {
return err
}
*m = append(*m, pm...)
*m = append(*m, matcher)
}
sort.Sort(labels.Matchers(*m))
return nil