Alerting: Deduplicate receivers during migration (#36812)

* Alerting: Deduplicate receivers during migration

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Fix review comments

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Better naming of contact points

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Handle some edge cases

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Don't add routes which point to default contact point

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

pkg/services/sqlstore/migrations/ualert/channel.go

@@ -5,6 +5,8 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"sort"
+	"strings"
 
 	"github.com/prometheus/alertmanager/pkg/labels"
 
@@ -85,21 +87,19 @@ func (m *migration) updateReceiverAndRoute(allChannels map[interface{}]*notifica
 		return err
 	}
 
-	amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
+	if recv != nil {
-	amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route)
+		amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
+	}
+	if route != nil {
+		amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route)
+	}
 
 	return nil
 }
 
 func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface{}, defaultChannels []*notificationChannel, allChannels map[interface{}]*notificationChannel) (*PostableApiReceiver, *Route, error) {
-	receiverName := getMigratedReceiverNameFromRuleUID(ruleUid)
-
 	portedChannels := []*PostableGrafanaReceiver{}
-	receiver := &PostableApiReceiver{
+	var receiver *PostableApiReceiver
-		Name: receiverName,
-	}
-
-	addedChannels := map[*notificationChannel]struct{}{}
 
 	addChannel := func(c *notificationChannel) error {
 		if c.Type == "hipchat" || c.Type == "sensu" {
@@ -122,31 +122,70 @@ func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface
 			Settings:              settings,
 			SecureSettings:        secureSettings,
 		})
-		addedChannels[c] = struct{}{}
 
 		return nil
 	}
 
-	for _, n := range channelUids {
+	// Remove obsolete notification channels.
-		c, ok := allChannels[n]
+	filteredChannelUids := make(map[interface{}]struct{})
-		if !ok {
+	for _, uid := range channelUids {
-			continue
+		_, ok := allChannels[uid]
+		if ok {
+			filteredChannelUids[uid] = struct{}{}
+		} else {
+			m.mg.Logger.Warn("ignoring obsolete notification channel", "uid", uid)
 		}
-
+	}
-		if err := addChannel(c); err != nil {
+	// Add default channels that are not obsolete.
-			return nil, nil, err
+	for _, c := range defaultChannels {
+		id := interface{}(c.Uid)
+		if c.Uid == "" {
+			id = c.ID
+		}
+		_, ok := allChannels[id]
+		if ok {
+			filteredChannelUids[id] = struct{}{}
 		}
 	}
 
-	for _, c := range defaultChannels {
+	if len(filteredChannelUids) == 0 && ruleUid != "default_route" {
-		if _, ok := addedChannels[c]; !ok {
+		// We use the default route instead. No need to add additional route.
-			if err := addChannel(c); err != nil {
+		return nil, nil, nil
+	}
+
+	chanKey, err := makeKeyForChannelGroup(filteredChannelUids)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var receiverName string
+	if rn, ok := m.portedChannelGroups[chanKey]; ok {
+		// We have ported these exact set of channels already. Re-use it.
+		receiverName = rn
+		if receiverName == "autogen-contact-point-default" {
+			// We don't need to create new routes if it's the default contact point.
+			return nil, nil, nil
+		}
+	} else {
+		for n := range filteredChannelUids {
+			if err := addChannel(allChannels[n]); err != nil {
 				return nil, nil, err
 			}
 		}
-	}
 
-	receiver.GrafanaManagedReceivers = portedChannels
+		if ruleUid == "default_route" {
+			receiverName = "autogen-contact-point-default"
+		} else {
+			m.lastReceiverID++
+			receiverName = fmt.Sprintf("autogen-contact-point-%d", m.lastReceiverID)
+		}
+
+		m.portedChannelGroups[chanKey] = receiverName
+		receiver = &PostableApiReceiver{
+			Name:                    receiverName,
+			GrafanaManagedReceivers: portedChannels,
+		}
+	}
 
 	n, v := getLabelForRouteMatching(ruleUid)
 	mat, err := labels.NewMatcher(labels.MatchEqual, n, v)
@@ -161,7 +200,45 @@ func (m *migration) makeReceiverAndRoute(ruleUid string, channelUids []interface
 	return receiver, route, nil
 }
 
-func (m *migration) updateDefaultAndUnmigratedChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error {
+// makeKeyForChannelGroup generates a unique for this group of channels UIDs.
+func makeKeyForChannelGroup(channelUids map[interface{}]struct{}) (string, error) {
+	uids := make([]string, 0, len(channelUids))
+	for u := range channelUids {
+		switch uid := u.(type) {
+		case string:
+			uids = append(uids, uid)
+		case int, int32, int64:
+			uids = append(uids, fmt.Sprintf("%d", uid))
+		default:
+			// Should never happen.
+			return "", fmt.Errorf("unknown channel UID type: %T", u)
+		}
+	}
+
+	sort.Strings(uids)
+	return strings.Join(uids, "::sep::"), nil
+}
+
+// addDefaultChannels should be called before adding any other routes.
+func (m *migration) addDefaultChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error {
+	// Default route and receiver.
+	recv, route, err := m.makeReceiverAndRoute("default_route", nil, defaultChannels, allChannels)
+	if err != nil {
+		return err
+	}
+
+	if recv != nil {
+		amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
+	}
+	if route != nil {
+		route.Matchers = nil // Don't need matchers for root route.
+		amConfig.AlertmanagerConfig.Route = route
+	}
+
+	return nil
+}
+
+func (m *migration) addUnmigratedChannels(amConfig *PostableUserConfig, allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel) error {
 	// Unmigrated channels.
 	portedChannels := []*PostableGrafanaReceiver{}
 	receiver := &PostableApiReceiver{
@@ -198,26 +275,6 @@ func (m *migration) updateDefaultAndUnmigratedChannels(amConfig *PostableUserCon
 		amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, receiver)
 	}
 
-	// Default route and receiver.
-	var channelUids = []interface{}{}
-	for _, c := range defaultChannels {
-		if c.Uid == "" {
-			channelUids = append(channelUids, c.ID)
-		} else {
-			channelUids = append(channelUids, c.Uid)
-		}
-	}
-
-	recv, route, err := m.makeReceiverAndRoute("default_route", channelUids, defaultChannels, allChannels)
-	if err != nil {
-		return err
-	}
-
-	route.Matchers = nil // Don't need matchers for root route.
-	route.Routes = amConfig.AlertmanagerConfig.Route.Routes
-	amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
-	amConfig.AlertmanagerConfig.Route = route
-
 	return nil
 }
 
@@ -275,10 +332,6 @@ func migrateSettingsToSecureSettings(chanType string, settings *simplejson.Json,
 	return settings, ss
 }
 
-func getMigratedReceiverNameFromRuleUID(ruleUID string) string {
-	return fmt.Sprintf("autogen-panel-recv-%s", ruleUID)
-}
-
 func getLabelForRouteMatching(ruleUID string) (string, string) {
 	return "rule_uid", ruleUID
 }

pkg/services/sqlstore/migrations/ualert/ualert.go

@@ -54,8 +54,9 @@ func AddDashAlertMigration(mg *migrator.Migrator) {
 			mg.Logger.Error("alert migration error: could not clear alert migration for removing data", "error", err)
 		}
 		mg.AddMigration(migTitle, &migration{
-			seenChannelUIDs:  make(map[string]struct{}),
+			seenChannelUIDs:     make(map[string]struct{}),
-			migratedChannels: make(map[*notificationChannel]struct{}),
+			migratedChannels:    make(map[*notificationChannel]struct{}),
+			portedChannelGroups: make(map[string]string),
 		})
 	case !ngEnabled && migrationRun:
 		// Remove the migration entry that creates unified alerting data. This is so when the feature
@@ -74,9 +75,11 @@ type migration struct {
 	sess *xorm.Session
 	mg   *migrator.Migrator
 
-	seenChannelUIDs  map[string]struct{}
+	seenChannelUIDs     map[string]struct{}
-	migratedChannels map[*notificationChannel]struct{}
+	migratedChannels    map[*notificationChannel]struct{}
-	silences         []*pb.MeshSilence
+	silences            []*pb.MeshSilence
+	portedChannelGroups map[string]string // Channel group key -> receiver name.
+	lastReceiverID      int               // For the auto generated receivers.
 }
 
 func (m *migration) SQL(dialect migrator.Dialect) string {
@@ -111,7 +114,10 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
 	}
 
 	amConfig := PostableUserConfig{}
-	amConfig.AlertmanagerConfig.Route = &Route{}
+	err = m.addDefaultChannels(&amConfig, allChannels, defaultChannels)
+	if err != nil {
+		return err
+	}
 
 	for _, da := range dashAlerts {
 		newCond, err := transConditions(*da.ParsedSettings, da.OrgId, dsIDMap)
@@ -245,7 +251,7 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
 	}
 
 	// Create a separate receiver for all the unmigrated channels.
-	err = m.updateDefaultAndUnmigratedChannels(&amConfig, allChannels, defaultChannels)
+	err = m.addUnmigratedChannels(&amConfig, allChannels, defaultChannels)
 	if err != nil {
 		return err
 	}