mirror of
https://github.com/grafana/grafana.git
synced 2024-11-25 18:30:41 -06:00
279 lines
10 KiB
Go
279 lines
10 KiB
Go
package notifier_test
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/grafana/grafana/pkg/infra/log"
|
|
"github.com/grafana/grafana/pkg/services/featuremgmt"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/models"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
|
|
"github.com/grafana/grafana/pkg/services/ngalert/remote"
|
|
remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client"
|
|
ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes"
|
|
"github.com/grafana/grafana/pkg/services/secrets/fakes"
|
|
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
|
|
"github.com/grafana/grafana/pkg/setting"
|
|
)
|
|
|
|
func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) {
|
|
ctx := context.Background()
|
|
nopLogger := log.NewNopLogger()
|
|
tenantID := "testTenantID"
|
|
password := "testPassword"
|
|
reg := prometheus.NewPedanticRegistry()
|
|
m := metrics.NewNGAlert(reg)
|
|
|
|
// We're gonna use an test server to send configuration and state to.
|
|
fakeAM := newFakeRemoteAlertmanager(t, tenantID, password)
|
|
testsrv := httptest.NewServer(fakeAM)
|
|
|
|
// We'll start with the default config and no values for silences and notifications.
|
|
kvStore := ngfakes.NewFakeKVStore(t)
|
|
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, ""))
|
|
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, ""))
|
|
configStore := notifier.NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{
|
|
1: {
|
|
OrgID: 1,
|
|
AlertmanagerConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
|
|
CreatedAt: time.Now().Unix(),
|
|
Default: true,
|
|
},
|
|
})
|
|
|
|
// Create the factory function for the MOA using the forked Alertmanager in remote secondary mode.
|
|
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
|
|
override := notifier.WithAlertmanagerOverride(func(factoryFn notifier.OrgAlertmanagerFactory) notifier.OrgAlertmanagerFactory {
|
|
return func(ctx context.Context, orgID int64) (notifier.Alertmanager, error) {
|
|
// Create internal Alertmanager.
|
|
internalAM, err := factoryFn(ctx, orgID)
|
|
require.NoError(t, err)
|
|
|
|
// Create remote Alertmanager.
|
|
externalAMCfg := remote.AlertmanagerConfig{
|
|
OrgID: 1,
|
|
URL: testsrv.URL,
|
|
TenantID: tenantID,
|
|
BasicAuthPassword: password,
|
|
DefaultConfig: setting.GetAlertmanagerDefaultConfiguration(),
|
|
}
|
|
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
|
|
remoteAM, err := remote.NewAlertmanager(externalAMCfg, notifier.NewFileStore(orgID, kvStore), secretsService.Decrypt, m)
|
|
require.NoError(t, err)
|
|
|
|
// Use both Alertmanager implementations in the forked Alertmanager.
|
|
cfg := remote.RemoteSecondaryConfig{
|
|
Logger: nopLogger,
|
|
OrgID: orgID,
|
|
Store: configStore,
|
|
// Note that we're setting a sync interval of 10 seconds.
|
|
SyncInterval: 10 * time.Second,
|
|
}
|
|
return remote.NewRemoteSecondaryForkedAlertmanager(cfg, internalAM, remoteAM)
|
|
}
|
|
})
|
|
|
|
cfg := &setting.Cfg{
|
|
DataPath: t.TempDir(),
|
|
UnifiedAlerting: setting.UnifiedAlertingSettings{
|
|
AlertmanagerConfigPollInterval: 3 * time.Minute,
|
|
DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
|
|
}, // do not poll in tests.
|
|
}
|
|
moa, err := notifier.NewMultiOrgAlertmanager(
|
|
cfg,
|
|
configStore,
|
|
notifier.NewFakeOrgStore(t, []int64{1}),
|
|
kvStore,
|
|
ngfakes.NewFakeProvisioningStore(),
|
|
secretsService.GetDecryptedValue,
|
|
m.GetMultiOrgAlertmanagerMetrics(),
|
|
nil,
|
|
nopLogger,
|
|
secretsService,
|
|
featuremgmt.WithFeatures(),
|
|
override,
|
|
)
|
|
require.NoError(t, err)
|
|
|
|
// It should send config and state on startup.
|
|
var lastConfig *remoteClient.UserGrafanaConfig
|
|
var lastState *remoteClient.UserGrafanaState
|
|
{
|
|
// We should start with no config and no state in the external Alertmanager.
|
|
require.Empty(t, fakeAM.config)
|
|
require.Empty(t, fakeAM.state)
|
|
|
|
// On the first sync (startup), both config and state should be updated.
|
|
require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx))
|
|
require.NotEmpty(t, fakeAM.config)
|
|
require.NotEmpty(t, fakeAM.state)
|
|
lastConfig, lastState = fakeAM.config, fakeAM.state
|
|
}
|
|
|
|
// It should send config and state on an interval.
|
|
{
|
|
// Let's change the configuration and state.
|
|
require.NoError(t, configStore.SaveAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{
|
|
AlertmanagerConfiguration: validConfig,
|
|
OrgID: 1,
|
|
LastApplied: time.Now().Unix(),
|
|
}))
|
|
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "lwEKhgEKATISFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCN2CkbAGEJbKrMsDIgwI7Z6RsAYQlsqsywMqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCO2ekbAGEJbKrMsDlwEKhgEKATESFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCN2CkbAGEJbKrMsDIgwI7Z6RsAYQlsqsywMqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCO2ekbAGEJbKrMsD"))
|
|
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "OgoqCgZncm91cDISEgoJcmVjZWl2ZXIyEgV0ZXN0MyoMCLSDkbAGEMvaofYCEgwIxJ+RsAYQy9qh9gI6CioKBmdyb3VwMRISCglyZWNlaXZlcjESBXRlc3QzKgwItIORsAYQy9qh9gISDAjEn5GwBhDL2qH2Ag=="))
|
|
|
|
// The sync interval (10s) has not elapsed yet, syncing should have no effect.
|
|
require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx))
|
|
require.Equal(t, fakeAM.config, lastConfig)
|
|
require.Equal(t, fakeAM.state, lastState)
|
|
|
|
// Syncing after the sync interval elapses should update both config and state.
|
|
require.Eventually(t, func() bool {
|
|
require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx))
|
|
return fakeAM.config != lastConfig && fakeAM.state != lastState
|
|
}, 15*time.Second, 300*time.Millisecond)
|
|
lastConfig, lastState = fakeAM.config, fakeAM.state
|
|
}
|
|
|
|
// It should send config and state on shutdown.
|
|
{
|
|
// Let's change the configuration and state again.
|
|
require.NoError(t, configStore.SaveAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{
|
|
AlertmanagerConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
|
|
Default: true,
|
|
OrgID: 1,
|
|
LastApplied: time.Now().Unix(),
|
|
}))
|
|
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "lwEKhgEKAWESFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCPuEkbAGEK3AhM8CIgwIi6GRsAYQrcCEzwIqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCIuhkbAGEK3AhM8ClwEKhgEKAWISFxIJYWxlcnRuYW1lGgp0ZXN0X2FsZXJ0EiMSDmdyYWZhbmFfZm9sZGVyGhF0ZXN0X2FsZXJ0X2ZvbGRlchoMCPuEkbAGEK3AhM8CIgwIi6GRsAYQrcCEzwIqCwiAkrjDmP7///8BQgxHcmFmYW5hIFRlc3RKDFRlc3QgU2lsZW5jZRIMCIuhkbAGEK3AhM8C"))
|
|
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "OAopCgZncm91cEESEgoJcmVjZWl2ZXJBEgV0ZXN0MyoLCNmEkbAGEOzO0BUSCwjpoJGwBhDsztAVOAopCgZncm91cEISEgoJcmVjZWl2ZXJCEgV0ZXN0MyoLCNmEkbAGEOzO0BUSCwjpoJGwBhDsztAV"))
|
|
|
|
// Both state and config should be updated when shutting the Alertmanager down.
|
|
moa.StopAndWait()
|
|
require.NotEqual(t, fakeAM.config, lastConfig)
|
|
require.NotEqual(t, fakeAM.state, lastState)
|
|
}
|
|
}
|
|
|
|
func newFakeRemoteAlertmanager(t *testing.T, user, pass string) *fakeRemoteAlertmanager {
|
|
return &fakeRemoteAlertmanager{
|
|
t: t,
|
|
username: user,
|
|
password: pass,
|
|
}
|
|
}
|
|
|
|
type fakeRemoteAlertmanager struct {
|
|
t *testing.T
|
|
config *remoteClient.UserGrafanaConfig
|
|
state *remoteClient.UserGrafanaState
|
|
username string
|
|
password string
|
|
}
|
|
|
|
// ServeHTTP handles all routes we need for getting and setting state and config.
|
|
func (f *fakeRemoteAlertmanager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Add("Content-Type", "application/json")
|
|
|
|
// Check that basic auth is in place.
|
|
user, pass, ok := r.BasicAuth()
|
|
require.True(f.t, ok)
|
|
require.Equal(f.t, f.username, user)
|
|
require.Equal(f.t, f.password, pass)
|
|
|
|
switch r.Method {
|
|
// GET routes
|
|
case http.MethodGet:
|
|
switch r.RequestURI {
|
|
case "/alertmanager/-/ready":
|
|
// Make the readiness check succeed.
|
|
w.WriteHeader(http.StatusOK)
|
|
case "/api/v1/grafana/config":
|
|
f.getConfig(w)
|
|
case "/api/v1/grafana/state":
|
|
f.getState(w)
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
|
|
// POST routes
|
|
case http.MethodPost:
|
|
switch r.RequestURI {
|
|
case "/api/v1/grafana/config":
|
|
f.postConfig(w, r)
|
|
case "/api/v1/grafana/state":
|
|
f.postState(w, r)
|
|
default:
|
|
w.WriteHeader(http.StatusNotFound)
|
|
}
|
|
}
|
|
}
|
|
|
|
type response struct {
|
|
Data any `json:"data"`
|
|
Status string `json:"status"`
|
|
}
|
|
|
|
func (f *fakeRemoteAlertmanager) postConfig(w http.ResponseWriter, r *http.Request) {
|
|
var cfg remoteClient.UserGrafanaConfig
|
|
require.NoError(f.t, json.NewDecoder(r.Body).Decode(&cfg))
|
|
|
|
f.config = &cfg
|
|
w.WriteHeader(http.StatusCreated)
|
|
require.NoError(f.t, json.NewEncoder(w).Encode(response{Status: "success"}))
|
|
}
|
|
|
|
func (f *fakeRemoteAlertmanager) getConfig(w http.ResponseWriter) {
|
|
res := response{
|
|
Data: f.config,
|
|
Status: "success",
|
|
}
|
|
require.NoError(f.t, json.NewEncoder(w).Encode(res))
|
|
}
|
|
|
|
func (f *fakeRemoteAlertmanager) postState(w http.ResponseWriter, r *http.Request) {
|
|
var state remoteClient.UserGrafanaState
|
|
require.NoError(f.t, json.NewDecoder(r.Body).Decode(&state))
|
|
|
|
f.state = &state
|
|
w.WriteHeader(http.StatusCreated)
|
|
require.NoError(f.t, json.NewEncoder(w).Encode(response{Status: "success"}))
|
|
}
|
|
|
|
func (f *fakeRemoteAlertmanager) getState(w http.ResponseWriter) {
|
|
res := response{
|
|
Data: f.state,
|
|
Status: "success",
|
|
}
|
|
require.NoError(f.t, json.NewEncoder(w).Encode(res))
|
|
}
|
|
|
|
var validConfig = `{
|
|
"template_files": {
|
|
"a": "template"
|
|
},
|
|
"alertmanager_config": {
|
|
"route": {
|
|
"receiver": "grafana-default-email"
|
|
},
|
|
"receivers": [{
|
|
"name": "grafana-default-email",
|
|
"grafana_managed_receiver_configs": [{
|
|
"uid": "",
|
|
"name": "email receiver",
|
|
"type": "email",
|
|
"settings": {
|
|
"addresses": "<example@email.com>"
|
|
}
|
|
}]
|
|
}]
|
|
}
|
|
}`
|