Alerting: Migrate paused alerts to silences (#34898)

* Alerting: Migrate paused alerts to silences

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Fix lint

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

go.mod

@@ -44,6 +44,7 @@ require (
 	github.com/go-sql-driver/mysql v1.6.0
 	github.com/go-stack/stack v1.8.0
 	github.com/gobwas/glob v0.2.3
+	github.com/gofrs/uuid v4.0.0+incompatible
 	github.com/golang/mock v1.5.0
 	github.com/google/go-cmp v0.5.5
 	github.com/google/uuid v1.2.0
@@ -68,6 +69,7 @@ require (
 	github.com/magefile/mage v1.11.0
 	github.com/mattn/go-isatty v0.0.12
 	github.com/mattn/go-sqlite3 v1.14.7
+	github.com/matttproud/golang_protobuf_extensions v1.0.1
 	github.com/opentracing/opentracing-go v1.2.0
 	github.com/patrickmn/go-cache v2.1.0+incompatible
 	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect

pkg/services/sqlstore/migrations/ualert/alert_rule.go

@@ -105,7 +105,10 @@ func (m *migration) makeAlertRule(cond condition, da dashAlert, folderUID string
 		For:             duration(da.For),
 		Updated:         time.Now().UTC(),
 		Annotations:     annotations,
-		Labels:          map[string]string{},
+		Labels: map[string]string{
+			"alertname": da.Name,
+			"message":   da.Message,
+		},
 	}
 
 	var err error
@@ -119,6 +122,14 @@ func (m *migration) makeAlertRule(cond condition, da dashAlert, folderUID string
 		return nil, err
 	}
 
+	// Label for routing and silences.
+	n, v := getLabelForRouteMatching(ar.Uid)
+	ar.Labels[n] = v
+
+	if err := m.addSilence(da, ar); err != nil {
+		m.mg.Logger.Error("alert migration error: failed to create silence", "rule_name", ar.Title, "err", err)
+	}
+
 	return ar, nil
 }
 

pkg/services/sqlstore/migrations/ualert/channel.go

@@ -67,9 +67,6 @@ func (m *migration) getNotificationChannelMap() (map[interface{}]*notificationCh
 }
 
 func (m *migration) updateReceiverAndRoute(allChannels map[interface{}]*notificationChannel, defaultChannels []*notificationChannel, da dashAlert, rule *alertRule, amConfig *PostableUserConfig) error {
-	rule.Labels["alertname"] = da.Name
-	rule.Annotations["message"] = da.Message
-
 	// Create receiver and route for this rule.
 	if allChannels == nil {
 		return nil
@@ -88,10 +85,6 @@ func (m *migration) updateReceiverAndRoute(allChannels map[interface{}]*notifica
 		return err
 	}
 
-	// Attach label for routing.
-	n, v := getLabelForRouteMatching(rule.Uid)
-	rule.Labels[n] = v
-
 	amConfig.AlertmanagerConfig.Receivers = append(amConfig.AlertmanagerConfig.Receivers, recv)
 	amConfig.AlertmanagerConfig.Route.Routes = append(amConfig.AlertmanagerConfig.Route.Routes, route)
 

pkg/services/sqlstore/migrations/ualert/dash_alert.go

@@ -14,6 +14,7 @@ type dashAlert struct {
 	Message     string
 	Frequency   int64
 	For         time.Duration
+	State       string
 
 	Settings       json.RawMessage
 	ParsedSettings *dashAlertSettings
@@ -30,6 +31,7 @@ SELECT id,
 	message,
 	frequency,
 	for,
+	state,
 	settings
 FROM
 	alert

pkg/services/sqlstore/migrations/ualert/silences.go

@@ -0,0 +1,107 @@
+package ualert
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/gofrs/uuid"
+	"github.com/matttproud/golang_protobuf_extensions/pbutil"
+	pb "github.com/prometheus/alertmanager/silence/silencepb"
+
+	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
+)
+
+func (m *migration) addSilence(da dashAlert, rule *alertRule) error {
+	if da.State != "paused" {
+		return nil
+	}
+
+	uid, err := uuid.NewV4()
+	if err != nil {
+		return errors.New("failed to create uuid for silence")
+	}
+
+	n, v := getLabelForRouteMatching(rule.Uid)
+	s := &pb.MeshSilence{
+		Silence: &pb.Silence{
+			Id: uid.String(),
+			Matchers: []*pb.Matcher{
+				{
+					Type:    pb.Matcher_EQUAL,
+					Name:    n,
+					Pattern: v,
+				},
+			},
+			StartsAt:  time.Now(),
+			EndsAt:    time.Now().Add(365 * 20 * time.Hour), // 1 year.
+			CreatedBy: "Grafana Migration",
+			Comment:   "Created during auto migration to unified alerting",
+		},
+		ExpiresAt: time.Now().Add(365 * 20 * time.Hour), // 1 year.
+	}
+
+	m.silences = append(m.silences, s)
+	return nil
+}
+
+func (m *migration) writeSilencesFile() error {
+	var buf bytes.Buffer
+	for _, e := range m.silences {
+		if _, err := pbutil.WriteDelimited(&buf, e); err != nil {
+			return err
+		}
+	}
+
+	f, err := openReplace(silencesFileName(m.mg))
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(f, bytes.NewReader(buf.Bytes())); err != nil {
+		return err
+	}
+
+	return f.Close()
+}
+
+func silencesFileName(mg *migrator.Migrator) string {
+	return filepath.Join(mg.Cfg.DataPath, "alerting", "silences")
+}
+
+// replaceFile wraps a file that is moved to another filename on closing.
+type replaceFile struct {
+	*os.File
+	filename string
+}
+
+func (f *replaceFile) Close() error {
+	if err := f.File.Sync(); err != nil {
+		return err
+	}
+	if err := f.File.Close(); err != nil {
+		return err
+	}
+	return os.Rename(f.File.Name(), f.filename)
+}
+
+// openReplace opens a new temporary file that is moved to filename on closing.
+func openReplace(filename string) (*replaceFile, error) {
+	tmpFilename := fmt.Sprintf("%s.%x", filename, uint64(rand.Int63()))
+
+	f, err := os.Create(tmpFilename)
+	if err != nil {
+		return nil, err
+	}
+
+	rf := &replaceFile{
+		File:     f,
+		filename: filename,
+	}
+	return rf, nil
+}

pkg/services/sqlstore/migrations/ualert/ualert.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"time"
 
+	pb "github.com/prometheus/alertmanager/silence/silencepb"
 	"xorm.io/xorm"
 
 	"github.com/grafana/grafana/pkg/services/sqlstore/migrator"
@@ -75,6 +76,7 @@ type migration struct {
 
 	seenChannelUIDs  map[string]struct{}
 	migratedChannels map[*notificationChannel]struct{}
+	silences         []*pb.MeshSilence
 }
 
 func (m *migration) SQL(dialect migrator.Dialect) string {
@@ -256,8 +258,15 @@ func (m *migration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
 		// the v1 config.
 		ConfigurationVersion: "v1",
 	})
-
+	if err != nil {
 		return err
+	}
+
+	if err := m.writeSilencesFile(); err != nil {
+		m.mg.Logger.Error("alert migration error: failed to write silence file", "err", err)
+	}
+
+	return nil
 }
 
 type AlertConfiguration struct {
@@ -307,5 +316,9 @@ func (m *rmMigration) Exec(sess *xorm.Session, mg *migrator.Migrator) error {
 		return err
 	}
 
+	if err := os.RemoveAll(silencesFileName(mg)); err != nil {
+		mg.Logger.Error("alert migration error: failed to remove silence file", "err", err)
+	}
+
 	return nil
 }