Alerting: Persist notification log and silences to the database (#39005)

* Alerting: Persist notification log and silences to the database

This removes the dependency of having persistent disk to run grafana alerting. Instead of regularly flushing the notification log and silences to disk we now flush the binary content of those files to the database encoded as a base64 string.

pkg/services/ngalert/ngalert.go

@@ -4,25 +4,25 @@ import (
 	"context"
 	"time"
 
-	"github.com/benbjohnson/clock"
-	"github.com/grafana/grafana/pkg/services/quota"
-	"golang.org/x/sync/errgroup"
-
-	"github.com/grafana/grafana/pkg/services/ngalert/api"
-	"github.com/grafana/grafana/pkg/services/ngalert/eval"
-	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
-	"github.com/grafana/grafana/pkg/services/ngalert/state"
-	"github.com/grafana/grafana/pkg/services/ngalert/store"
-
 	"github.com/grafana/grafana/pkg/api/routing"
+	"github.com/grafana/grafana/pkg/infra/kvstore"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/datasourceproxy"
 	"github.com/grafana/grafana/pkg/services/datasources"
+	"github.com/grafana/grafana/pkg/services/ngalert/api"
+	"github.com/grafana/grafana/pkg/services/ngalert/eval"
+	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/notifier"
 	"github.com/grafana/grafana/pkg/services/ngalert/schedule"
+	"github.com/grafana/grafana/pkg/services/ngalert/state"
+	"github.com/grafana/grafana/pkg/services/ngalert/store"
+	"github.com/grafana/grafana/pkg/services/quota"
 	"github.com/grafana/grafana/pkg/services/sqlstore"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/tsdb"
+
+	"github.com/benbjohnson/clock"
+	"golang.org/x/sync/errgroup"
 )
 
 const (
@@ -38,13 +38,14 @@ const (
 )
 
 func ProvideService(cfg *setting.Cfg, dataSourceCache datasources.CacheService, routeRegister routing.RouteRegister,
-	sqlStore *sqlstore.SQLStore, dataService *tsdb.Service, dataProxy *datasourceproxy.DataSourceProxyService,
+	sqlStore *sqlstore.SQLStore, kvStore kvstore.KVStore, dataService *tsdb.Service, dataProxy *datasourceproxy.DataSourceProxyService,
 	quotaService *quota.QuotaService, m *metrics.Metrics) (*AlertNG, error) {
 	ng := &AlertNG{
 		Cfg:             cfg,
 		DataSourceCache: dataSourceCache,
 		RouteRegister:   routeRegister,
 		SQLStore:        sqlStore,
+		KVStore:         kvStore,
 		DataService:     dataService,
 		DataProxy:       dataProxy,
 		QuotaService:    quotaService,
@@ -69,6 +70,7 @@ type AlertNG struct {
 	DataSourceCache datasources.CacheService
 	RouteRegister   routing.RouteRegister
 	SQLStore        *sqlstore.SQLStore
+	KVStore         kvstore.KVStore
 	DataService     *tsdb.Service
 	DataProxy       *datasourceproxy.DataSourceProxyService
 	QuotaService    *quota.QuotaService
@@ -95,7 +97,7 @@ func (ng *AlertNG) init() error {
 		Logger:                 ng.Log,
 	}
 
-	ng.MultiOrgAlertmanager = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store)
+	ng.MultiOrgAlertmanager = notifier.NewMultiOrgAlertmanager(ng.Cfg, store, store, ng.KVStore)
 
 	// Let's make sure we're able to complete an initial sync of Alertmanagers before we start the alerting components.
 	if err := ng.MultiOrgAlertmanager.LoadAndSyncAlertmanagersForOrgs(context.Background()); err != nil {

pkg/services/ngalert/notifier/alertmanager.go

@@ -27,6 +27,7 @@ import (
 	"github.com/prometheus/common/model"
 
 	"github.com/grafana/grafana/pkg/components/securejsondata"
+	"github.com/grafana/grafana/pkg/infra/kvstore"
 	"github.com/grafana/grafana/pkg/infra/log"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/logging"
@@ -38,6 +39,9 @@ import (
 )
 
 const (
+	notificationLogFilename = "notifications"
+	silencesFilename        = "silences"
+
 	workingDir = "alerting"
 	// How long should we keep silences and notification entries on-disk after they've served their purpose.
 	retentionNotificationsAndSilences = 5 * 24 * time.Hour
@@ -77,9 +81,10 @@ type Alertmanager struct {
 	logger      log.Logger
 	gokitLogger gokit_log.Logger
 
-	Settings *setting.Cfg
-	Store    store.AlertingStore
-	Metrics  *metrics.Metrics
+	Settings  *setting.Cfg
+	Store     store.AlertingStore
+	fileStore *FileStore
+	Metrics   *metrics.Metrics
 
 	notificationLog *nflog.Log
 	marker          types.Marker
@@ -106,28 +111,39 @@ type Alertmanager struct {
 	orgID           int64
 }
 
-func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, m *metrics.Metrics) (*Alertmanager, error) {
+func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, kvStore kvstore.KVStore, m *metrics.Metrics) (*Alertmanager, error) {
 	am := &Alertmanager{
 		Settings:          cfg,
 		stopc:             make(chan struct{}),
 		logger:            log.New("alertmanager", "org", orgID),
 		marker:            types.NewMarker(m.Registerer),
 		stageMetrics:      notify.NewMetrics(m.Registerer),
-		dispatcherMetrics: dispatch.NewDispatcherMetrics(m.Registerer),
+		dispatcherMetrics: dispatch.NewDispatcherMetrics(false, m.Registerer),
 		Store:             store,
 		Metrics:           m,
 		orgID:             orgID,
 	}
 
 	am.gokitLogger = gokit_log.NewLogfmtLogger(logging.NewWrapper(am.logger))
+	am.fileStore = NewFileStore(am.orgID, kvStore, am.WorkingDirPath())
+
+	nflogFilepath, err := am.fileStore.FilepathFor(context.TODO(), notificationLogFilename)
+	if err != nil {
+		return nil, err
+	}
+	silencesFilePath, err := am.fileStore.FilepathFor(context.TODO(), silencesFilename)
+	if err != nil {
+		return nil, err
+	}
 
 	// Initialize the notification log
 	am.wg.Add(1)
-	var err error
 	am.notificationLog, err = nflog.New(
 		nflog.WithRetention(retentionNotificationsAndSilences),
-		nflog.WithSnapshot(filepath.Join(am.WorkingDirPath(), "notifications")),
-		nflog.WithMaintenance(maintenanceNotificationAndSilences, am.stopc, am.wg.Done),
+		nflog.WithSnapshot(nflogFilepath),
+		nflog.WithMaintenance(maintenanceNotificationAndSilences, am.stopc, am.wg.Done, func() (int64, error) {
+			return am.fileStore.Persist(context.TODO(), notificationLogFilename, am.notificationLog)
+		}),
 	)
 	if err != nil {
 		return nil, fmt.Errorf("unable to initialize the notification log component of alerting: %w", err)
@@ -135,7 +151,7 @@ func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, m
 	// Initialize silences
 	am.silences, err = silence.New(silence.Options{
 		Metrics:      m.Registerer,
-		SnapshotFile: filepath.Join(am.WorkingDirPath(), "silences"),
+		SnapshotFile: silencesFilePath,
 		Retention:    retentionNotificationsAndSilences,
 	})
 	if err != nil {
@@ -144,12 +160,14 @@ func newAlertmanager(orgID int64, cfg *setting.Cfg, store store.AlertingStore, m
 
 	am.wg.Add(1)
 	go func() {
-		am.silences.Maintenance(15*time.Minute, filepath.Join(am.WorkingDirPath(), "silences"), am.stopc)
+		am.silences.Maintenance(15*time.Minute, silencesFilePath, am.stopc, func() (int64, error) {
+			return am.fileStore.Persist(context.TODO(), silencesFilename, am.silences)
+		})
 		am.wg.Done()
 	}()
 
 	// Initialize in-memory alerts
-	am.alerts, err = mem.NewAlerts(context.Background(), am.marker, memoryAlertsGCInterval, am.gokitLogger)
+	am.alerts, err = mem.NewAlerts(context.Background(), am.marker, memoryAlertsGCInterval, nil, am.gokitLogger)
 	if err != nil {
 		return nil, fmt.Errorf("unable to initialize the alert provider component of alerting: %w", err)
 	}
@@ -390,7 +408,7 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig
 	}
 
 	am.route = dispatch.NewRoute(cfg.AlertmanagerConfig.Route, nil)
-	am.dispatcher = dispatch.NewDispatcher(am.alerts, am.route, routingStage, am.marker, timeoutFunc, am.gokitLogger, am.dispatcherMetrics)
+	am.dispatcher = dispatch.NewDispatcher(am.alerts, am.route, routingStage, am.marker, timeoutFunc, &nilLimits{}, am.gokitLogger, am.dispatcherMetrics)
 
 	am.wg.Add(1)
 	go func() {
@@ -707,3 +725,7 @@ func timeoutFunc(d time.Duration) time.Duration {
 	}
 	return d + waitFunc()
 }
+
+type nilLimits struct{}
+
+func (n nilLimits) MaxNumberOfAggregationGroups() int { return 0 }

pkg/services/ngalert/notifier/alertmanager_test.go

@@ -47,7 +47,8 @@ func setupAMTest(t *testing.T) *Alertmanager {
 		Logger:                 log.New("alertmanager-test"),
 	}
 
-	am, err := newAlertmanager(1, cfg, store, m)
+	kvStore := newFakeKVStore(t)
+	am, err := newAlertmanager(1, cfg, store, kvStore, m)
 	require.NoError(t, err)
 	return am
 }
@@ -310,7 +311,7 @@ func TestPutAlert(t *testing.T) {
 		t.Run(c.title, func(t *testing.T) {
 			r := prometheus.NewRegistry()
 			am.marker = types.NewMarker(r)
-			am.alerts, err = mem.NewAlerts(context.Background(), am.marker, 15*time.Minute, gokit_log.NewLogfmtLogger(logging.NewWrapper(am.logger)))
+			am.alerts, err = mem.NewAlerts(context.Background(), am.marker, 15*time.Minute, nil, gokit_log.NewLogfmtLogger(logging.NewWrapper(am.logger)))
 			require.NoError(t, err)
 
 			alerts := []*types.Alert{}

pkg/services/ngalert/notifier/file_store.go

@@ -0,0 +1,109 @@
+package notifier
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"github.com/grafana/grafana/pkg/infra/kvstore"
+)
+
+const KVNamespace = "alertmanager"
+
+// State represents any of the two 'states' of the alertmanager. Notification log or Silences.
+// MarshalBinary returns the binary representation of this internal state based on the protobuf.
+type State interface {
+	MarshalBinary() ([]byte, error)
+}
+
+// FileStore is in charge of persisting the alertmanager files to the database.
+// It uses the KVstore table and encodes the files as a base64 string.
+type FileStore struct {
+	kv             *kvstore.NamespacedKVStore
+	orgID          int64
+	workingDirPath string
+}
+
+func NewFileStore(orgID int64, store kvstore.KVStore, workingDirPath string) *FileStore {
+	return &FileStore{
+		workingDirPath: workingDirPath,
+		orgID:          orgID,
+		kv:             kvstore.WithNamespace(store, orgID, KVNamespace),
+	}
+}
+
+// FilepathFor returns the filepath to an Alertmanager file.
+// If the file is already present on disk it no-ops.
+// If not, it tries to read the database and if there's no file it no-ops.
+// If there is a file in the database, it decodes it and writes to disk for Alertmanager consumption.
+func (fs *FileStore) FilepathFor(ctx context.Context, filename string) (string, error) {
+	// If a file is already present, we'll use that one and eventually save it to the database.
+	// We don't need to do anything else.
+	if fs.IsExists(filename) {
+		return fs.pathFor(filename), nil
+	}
+
+	// Then, let's attempt to read it from the database.
+	content, exists, err := fs.kv.Get(ctx, filename)
+	if err != nil {
+		return "", fmt.Errorf("error reading file '%s' from database: %w", filename, err)
+	}
+
+	// if it doesn't exist, let's no-op and let the Alertmanager create one. We'll eventually save it to the database.
+	if !exists {
+		return fs.pathFor(filename), nil
+	}
+
+	// If we have a file stored in the database, let's decode it and write it to disk to perform that initial load to memory.
+	bytes, err := decode(content)
+	if err != nil {
+		return "", fmt.Errorf("error decoding file '%s': %w", filename, err)
+	}
+
+	if err := fs.WriteFileToDisk(filename, bytes); err != nil {
+		return "", fmt.Errorf("error writing file %s: %w", filename, err)
+	}
+
+	return fs.pathFor(filename), err
+}
+
+// Persist takes care of persisting the binary representation of internal state to the database as a base64 encoded string.
+func (fs *FileStore) Persist(ctx context.Context, filename string, st State) (int64, error) {
+	var size int64
+
+	bytes, err := st.MarshalBinary()
+	if err != nil {
+		return size, err
+	}
+
+	if err = fs.kv.Set(ctx, filename, encode(bytes)); err != nil {
+		return size, err
+	}
+
+	return int64(len(bytes)), err
+}
+
+// IsExists verifies if the file exists or not.
+func (fs *FileStore) IsExists(fn string) bool {
+	_, err := os.Stat(fs.pathFor(fn))
+	return os.IsExist(err)
+}
+
+// WriteFileToDisk writes a file with the provided name and contents to the Alertmanager working directory with the default grafana permission.
+func (fs *FileStore) WriteFileToDisk(fn string, content []byte) error {
+	return os.WriteFile(fs.pathFor(fn), content, 0644)
+}
+
+func (fs *FileStore) pathFor(fn string) string {
+	return filepath.Join(fs.workingDirPath, fn)
+}
+
+func decode(s string) ([]byte, error) {
+	return base64.StdEncoding.DecodeString(s)
+}
+
+func encode(b []byte) string {
+	return base64.StdEncoding.EncodeToString(b)
+}

pkg/services/ngalert/notifier/file_store_test.go

@@ -0,0 +1,74 @@
+package notifier
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestFileStore_FilepathFor(t *testing.T) {
+	store := newFakeKVStore(t)
+	workingDir := t.TempDir()
+	fs := NewFileStore(1, store, workingDir)
+	filekey := "silences"
+	filePath := filepath.Join(workingDir, filekey)
+
+	// With a file already on disk, it returns the existing file's filepath and no modification to the original file.
+	{
+		require.NoError(t, os.WriteFile(filePath, []byte("silence1,silence2"), 0644))
+		r, err := fs.FilepathFor(context.Background(), filekey)
+		require.NoError(t, err)
+		require.Equal(t, filePath, r)
+		f, err := ioutil.ReadFile(filepath.Clean(filePath))
+		require.NoError(t, err)
+		require.Equal(t, "silence1,silence2", string(f))
+		require.NoError(t, os.Remove(filePath))
+	}
+
+	// With a file already on the database, it writes the file to disk and returns the filepath.
+	{
+		require.NoError(t, store.Set(context.Background(), 1, KVNamespace, filekey, encode([]byte("silence1,silence3"))))
+		r, err := fs.FilepathFor(context.Background(), filekey)
+		require.NoError(t, err)
+		require.Equal(t, filePath, r)
+		f, err := ioutil.ReadFile(filepath.Clean(filePath))
+		require.NoError(t, err)
+		require.Equal(t, "silence1,silence3", string(f))
+		require.NoError(t, os.Remove(filePath))
+		require.NoError(t, store.Del(context.Background(), 1, KVNamespace, filekey))
+	}
+
+	// With no file on disk or database, it returns the original filepath.
+	{
+		r, err := fs.FilepathFor(context.Background(), filekey)
+		require.NoError(t, err)
+		require.Equal(t, filePath, r)
+		_, err = ioutil.ReadFile(filepath.Clean(filePath))
+		require.Error(t, err)
+	}
+}
+
+func TestFileStore_Persist(t *testing.T) {
+	store := newFakeKVStore(t)
+	state := &fakeState{data: "something to marshal"}
+	workingDir := t.TempDir()
+	fs := NewFileStore(1, store, workingDir)
+	filekey := "silences"
+
+	size, err := fs.Persist(context.Background(), filekey, state)
+	require.NoError(t, err)
+	require.Equal(t, int64(20), size)
+	store.mtx.Lock()
+	require.Len(t, store.store, 1)
+	store.mtx.Unlock()
+	v, ok, err := store.Get(context.Background(), 1, KVNamespace, filekey)
+	require.NoError(t, err)
+	require.True(t, ok)
+	b, err := decode(v)
+	require.NoError(t, err)
+	require.Equal(t, "something to marshal", string(b))
+}

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -6,6 +6,7 @@ import (
 	"sync"
 	"time"
 
+	"github.com/grafana/grafana/pkg/infra/kvstore"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -13,10 +14,7 @@ import (
 )
 
 var (
-	SyncOrgsPollInterval = 1 * time.Minute
-)
-
-var (
+	SyncOrgsPollInterval    = 1 * time.Minute
 	ErrNoAlertmanagerForOrg = fmt.Errorf("Alertmanager does not exist for this organization")
 	ErrAlertmanagerNotReady = fmt.Errorf("Alertmanager is not ready yet")
 )
@@ -30,17 +28,19 @@ type MultiOrgAlertmanager struct {
 
 	configStore store.AlertingStore
 	orgStore    store.OrgStore
+	kvStore     kvstore.KVStore
 
 	orgRegistry *metrics.OrgRegistries
 }
 
-func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore) *MultiOrgAlertmanager {
+func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, orgStore store.OrgStore, kvStore kvstore.KVStore) *MultiOrgAlertmanager {
 	return &MultiOrgAlertmanager{
 		settings:      cfg,
 		logger:        log.New("multiorg.alertmanager"),
 		alertmanagers: map[int64]*Alertmanager{},
 		configStore:   configStore,
 		orgStore:      orgStore,
+		kvStore:       kvStore,
 		orgRegistry:   metrics.NewOrgRegistries(),
 	}
 }
@@ -86,7 +86,7 @@ func (moa *MultiOrgAlertmanager) SyncAlertmanagersForOrgs(orgIDs []int64) {
 		existing, found := moa.alertmanagers[orgID]
 		if !found {
 			reg := moa.orgRegistry.GetOrCreateOrgRegistry(orgID)
-			am, err := newAlertmanager(orgID, moa.settings, moa.configStore, metrics.NewMetrics(reg))
+			am, err := newAlertmanager(orgID, moa.settings, moa.configStore, moa.kvStore, metrics.NewMetrics(reg))
 			if err != nil {
 				moa.logger.Error("unable to create Alertmanager for org", "org", orgID, "err", err)
 			}

pkg/services/ngalert/notifier/multiorg_alertmanager_test.go

@@ -19,7 +19,8 @@ func TestMultiOrgAlertmanager_SyncAlertmanagersForOrgs(t *testing.T) {
 		orgs: []int64{1, 2, 3},
 	}
 	SyncOrgsPollInterval = 10 * time.Minute // Don't poll in unit tests.
-	mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore)
+	kvStore := newFakeKVStore(t)
+	mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore, kvStore)
 	ctx := context.Background()
 
 	// Ensure that one Alertmanager is created per org.
@@ -50,7 +51,8 @@ func TestMultiOrgAlertmanager_AlertmanagerFor(t *testing.T) {
 	}
 
 	SyncOrgsPollInterval = 10 * time.Minute // Don't poll in unit tests.
-	mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore)
+	kvStore := newFakeKVStore(t)
+	mam := NewMultiOrgAlertmanager(&setting.Cfg{}, configStore, orgStore, kvStore)
 	ctx := context.Background()
 
 	// Ensure that one Alertmanagers is created per org.

pkg/services/ngalert/notifier/testing.go

@@ -2,6 +2,8 @@ package notifier
 
 import (
 	"context"
+	"sync"
+	"testing"
 
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
@@ -54,3 +56,76 @@ type FakeOrgStore struct {
 func (f *FakeOrgStore) GetOrgs(_ context.Context) ([]int64, error) {
 	return f.orgs, nil
 }
+
+type FakeKVStore struct {
+	mtx   sync.Mutex
+	store map[int64]map[string]map[string]string
+}
+
+func newFakeKVStore(t *testing.T) *FakeKVStore {
+	t.Helper()
+
+	return &FakeKVStore{
+		store: map[int64]map[string]map[string]string{},
+	}
+}
+
+func (fkv *FakeKVStore) Get(_ context.Context, orgId int64, namespace string, key string) (string, bool, error) {
+	fkv.mtx.Lock()
+	defer fkv.mtx.Unlock()
+	org, ok := fkv.store[orgId]
+	if !ok {
+		return "", false, nil
+	}
+	k, ok := org[namespace]
+	if !ok {
+		return "", false, nil
+	}
+
+	v, ok := k[key]
+	if !ok {
+		return "", false, nil
+	}
+
+	return v, true, nil
+}
+func (fkv *FakeKVStore) Set(_ context.Context, orgId int64, namespace string, key string, value string) error {
+	fkv.mtx.Lock()
+	defer fkv.mtx.Unlock()
+	org, ok := fkv.store[orgId]
+	if !ok {
+		fkv.store[orgId] = map[string]map[string]string{}
+	}
+	_, ok = org[namespace]
+	if !ok {
+		fkv.store[orgId][namespace] = map[string]string{}
+	}
+
+	fkv.store[orgId][namespace][key] = value
+
+	return nil
+}
+func (fkv *FakeKVStore) Del(_ context.Context, orgId int64, namespace string, key string) error {
+	fkv.mtx.Lock()
+	defer fkv.mtx.Unlock()
+	org, ok := fkv.store[orgId]
+	if !ok {
+		return nil
+	}
+	_, ok = org[namespace]
+	if !ok {
+		return nil
+	}
+
+	delete(fkv.store[orgId][namespace], key)
+
+	return nil
+}
+
+type fakeState struct {
+	data string
+}
+
+func (fs *fakeState) MarshalBinary() ([]byte, error) {
+	return []byte(fs.data), nil
+}

pkg/services/ngalert/schedule/schedule_unit_test.go

@@ -238,7 +238,7 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac
 		RuleStore:               rs,
 		InstanceStore:           is,
 		AdminConfigStore:        acs,
-		MultiOrgNotifier:        notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, &notifier.FakeConfigStore{}, &notifier.FakeOrgStore{}),
+		MultiOrgNotifier:        notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, &notifier.FakeConfigStore{}, &notifier.FakeOrgStore{}, &notifier.FakeKVStore{}),
 		Logger:                  logger,
 		Metrics:                 metrics.NewMetrics(prometheus.NewRegistry()),
 		AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.

pkg/services/ngalert/tests/util.go

@@ -35,8 +35,7 @@ func SetupTestEnv(t *testing.T, baseInterval time.Duration) (*ngalert.AlertNG, *
 	cfg.FeatureToggles = map[string]bool{"ngalert": true}
 
 	m := metrics.NewMetrics(prometheus.NewRegistry())
-	ng, err := ngalert.ProvideService(cfg, nil, routing.NewRouteRegister(), sqlstore.InitTestDB(t), nil, nil, nil,
-		m)
+	ng, err := ngalert.ProvideService(cfg, nil, routing.NewRouteRegister(), sqlstore.InitTestDB(t), nil, nil, nil, nil, m)
 	require.NoError(t, err)
 	return ng, &store.DBstore{
 		SQLStore:     ng.SQLStore,