IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Alerting: Deduplicate receivers during migration (#36812)

Browse Source 
* Alerting: Deduplicate receivers during migration

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Fix review comments

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Better naming of contact points

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Handle some edge cases

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Don't add routes which point to default contact point

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>
This commit is contained in:
Ganesh Vernekar 2021-07-29 19:26:19 +05:30 committed by GitHub
parent 2bf2f30e97
commit 10f8a138ae
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 113 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
pkg/services/sqlstore/migrations/ualert/channel.go
View File

@ -5,6 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"sort"
"strings"
"github.com/prometheus/alertmanager/pkg/labels"
@ -85,21 +87,19 @@ func (m *migration) updateReceiverAndRoute(allChannels map[interface{}]*notifica
return err
}
amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route)
if recv != nil {
amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
}
if route != nil {
amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route)
}
return nil
}
func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface{}, defaultChannels []*notificationChannel, allChannels map[interface{}]*notificationChannel) (*PostableApiReceiver, *Route, error) {
receiverName := getMigratedReceiverNameFromRuleUID(ruleUid)
portedChannels := []*PostableGrafanaReceiver{}
receiver := &PostableApiReceiver{
Name: receiverName,
}
addedChannels := map[*notificationChannel]struct{}{}
var receiver *PostableApiReceiver
addChannel := func(c *notificationChannel) error {
if c.Type == "hipchat" || c.Type == "sensu" {
@ -122,31 +122,70 @@ func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface
Settings: settings,
SecureSettings: secureSettings,
})
addedChannels[c] = struct{}{}
return nil
}
for _, n := range channelUids {
c, ok := allChannels[n]
if !ok {
continue
// Remove obsolete notification channels.
filteredChannelUids := make(map[interface{}]struct{})
for _, uid := range channelUids {
_, ok := allChannels[uid]
if ok {
filteredChannelUids[uid] = struct{}{}
} else {
m.mg.Logger.Warn("ignoring obsolete notification channel", "uid", uid)
}
if err := addChannel(c); err != nil {
return nil, nil, err
}
// Add default channels that are not obsolete.
for _, c := range defaultChannels {
id := interface{}(c.Uid)
if c.Uid == "" {
id = c.ID
}
_, ok := allChannels[id]
if ok {
filteredChannelUids[id] = struct{}{}
}
}
for _, c := range defaultChannels {
if _, ok := addedChannels[c]; !ok {
if err := addChannel(c); err != nil {
if len(filteredChannelUids) == 0 && ruleUid != "default_route" {
// We use the default route instead. No need to add additional route.
return nil, nil, nil
}
chanKey, err := makeKeyForChannelGroup(filteredChannelUids)
if err != nil {
return nil, nil, err
}
var receiverName string
if rn, ok := m.portedChannelGroups[chanKey]; ok {
// We have ported these exact set of channels already. Re-use it.
receiverName = rn
if receiverName == "autogen-contact-point-default" {
// We don't need to create new routes if it's the default contact point.
return nil, nil, nil
}
} else {
for n := range filteredChannelUids {
if err := addChannel(allChannels[n]); err != nil {
return nil, nil, err
}
}
}
receiver.GrafanaManagedReceivers = portedChannels
if ruleUid == "default_route" {
receiverName = "autogen-contact-point-default"
} else {
m.lastReceiverID++
receiverName = fmt.Sprintf("autogen-contact-point-%d", m.lastReceiverID)
}
m.portedChannelGroups[chanKey] = receiverName
receiver = &PostableApiReceiver{
Name: receiverName,
GrafanaManagedReceivers: portedChannels,
}
}
n, v := getLabelForRouteMatching(ruleUid)
mat, err := labels.NewMatcher(labels.MatchEqual, n, v)
@ -161,7 +200,45 @@ func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface
return receiver, route, nil
}
func (m *migration) updateDefaultAndUnmigratedChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error {
// makeKeyForChannelGroup generates a unique for this group of channels UIDs.
func makeKeyForChannelGroup(channelUids map[interface{}]struct{}) (string, error) {
uids := make([]string, 0, len(channelUids))
for u := range channelUids {
switch uid := u.(type) {
case string:
uids = append(uids, uid)
case int, int32, int64:
uids = append(uids, fmt.Sprintf("%d", uid))
default:
// Should never happen.
return "", fmt.Errorf("unknown channel UID type: %T", u)
}
}
sort.Strings(uids)
return strings.Join(uids, "::sep::"), nil
}
// addDefaultChannels should be called before adding any other routes.
func (m *migration) addDefaultChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error {
// Default route and receiver.
recv, route, err := m.makeReceiverAndRoute("default_route", nil, defaultChannels, allChannels)
if err != nil {
return err
}
if recv != nil {
amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
}
if route != nil {
route.Matchers = nil // Don't need matchers for root route.
amConfig.AlertmanagerConfig.Route = route
}
return nil
}
func (m *migration) addUnmigratedChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error {
// Unmigrated channels.
portedChannels := []*PostableGrafanaReceiver{}
receiver := &PostableApiReceiver{
@ -198,26 +275,6 @@ func (m *migration) updateDefaultAndUnmigratedChannels(amConfig *PostableUserCon
amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, receiver)
}
// Default route and receiver.
var channelUids = []interface{}{}
for _, c := range defaultChannels {
if c.Uid == "" {
channelUids = append(channelUids, c.ID)
} else {
channelUids = append(channelUids, c.Uid)
}
}
recv, route, err := m.makeReceiverAndRoute("default_route", channelUids, defaultChannels, allChannels)
if err != nil {
return err
}
route.Matchers = nil // Don't need matchers for root route.
route.Routes = amConfig.AlertmanagerConfig.Route.Routes
amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
amConfig.AlertmanagerConfig.Route = route
return nil
}
@ -275,10 +332,6 @@ func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json,
return settings, ss
}
func getMigratedReceiverNameFromRuleUID(ruleUID string) string {
return fmt.Sprintf("autogen-panel-recv-%s", ruleUID)
}
func getLabelForRouteMatching(ruleUID string) (string, string) {
return "rule_uid", ruleUID
}

20
pkg/services/sqlstore/migrations/ualert/ualert.go
View File

@ -54,8 +54,9 @@ func AddDashAlertMigration(mg *migrator.Migrator) {
mg.Logger.Error("alert migration error: could not clear alert migration for removing data", "error", err)
}
mg.AddMigration(migTitle, &migration{
seenChannelUIDs: make(map[string]struct{}),
migratedChannels: make(map[*notificationChannel]struct{}),
seenChannelUIDs: make(map[string]struct{}),
migratedChannels: make(map[*notificationChannel]struct{}),
portedChannelGroups: make(map[string]string),
})
case !ngEnabled && migrationRun:
// Remove the migration entry that creates unified alerting data. This is so when the feature
@ -74,9 +75,11 @@ type migration struct {
sess *xorm.Session
mg *migrator.Migrator
seenChannelUIDs map[string]struct{}
migratedChannels map[*notificationChannel]struct{}
silences []*pb.MeshSilence
seenChannelUIDs map[string]struct{}
migratedChannels map[*notificationChannel]struct{}
silences []*pb.MeshSilence
portedChannelGroups map[string]string // Channel group key -> receiver name.
lastReceiverID int // For the auto generated receivers.
}
func (m *migration) SQL(dialect migrator.Dialect) string {
@ -111,7 +114,10 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
}
amConfig := PostableUserConfig{}
amConfig.AlertmanagerConfig.Route = &Route{}
err = m.addDefaultChannels(&amConfig, allChannels, defaultChannels)
if err != nil {
return err
}
for _, da := range dashAlerts {
newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap)
@ -245,7 +251,7 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
}
// Create a separate receiver for all the unmigrated channels.
err = m.updateDefaultAndUnmigratedChannels(&amConfig, allChannels, defaultChannels)
err = m.addUnmigratedChannels(&amConfig, allChannels, defaultChannels)
if err != nil {
return err
}