Alerting: Manage remote Alertmanager silences (#75452)

* Alerting: Manage remote Alertmanager silences

* fix typo

* check errors when encoding json in fake external AM

* take path from configured URL, check for nil responses

pkg/services/ngalert/api/api_alertmanager.go

@@ -75,7 +75,7 @@ func (srv AlertmanagerSrv) RouteCreateSilence(c *contextmodel.ReqContext, postab
 		return ErrResp(http.StatusUnauthorized, fmt.Errorf("user is not authorized to %s silences", errAction), "")
 	}
 
-	silenceID, err := am.CreateSilence(&postableSilence)
+	silenceID, err := am.CreateSilence(c.Req.Context(), &postableSilence)
 	if err != nil {
 		if errors.Is(err, alertingNotify.ErrSilenceNotFound) {
 			return ErrResp(http.StatusNotFound, err, "")
@@ -112,7 +112,7 @@ func (srv AlertmanagerSrv) RouteDeleteSilence(c *contextmodel.ReqContext, silenc
 		return errResp
 	}
 
-	if err := am.DeleteSilence(silenceID); err != nil {
+	if err := am.DeleteSilence(c.Req.Context(), silenceID); err != nil {
 		if errors.Is(err, alertingNotify.ErrSilenceNotFound) {
 			return ErrResp(http.StatusNotFound, err, "")
 		}
@@ -199,7 +199,7 @@ func (srv AlertmanagerSrv) RouteGetSilence(c *contextmodel.ReqContext, silenceID
 		return errResp
 	}
 
-	gettableSilence, err := am.GetSilence(silenceID)
+	gettableSilence, err := am.GetSilence(c.Req.Context(), silenceID)
 	if err != nil {
 		if errors.Is(err, alertingNotify.ErrSilenceNotFound) {
 			return ErrResp(http.StatusNotFound, err, "")
@@ -216,7 +216,7 @@ func (srv AlertmanagerSrv) RouteGetSilences(c *contextmodel.ReqContext) response
 		return errResp
 	}
 
-	gettableSilences, err := am.ListSilences(c.QueryStrings("filter"))
+	gettableSilences, err := am.ListSilences(c.Req.Context(), c.QueryStrings("filter"))
 	if err != nil {
 		if errors.Is(err, alertingNotify.ErrListSilencesBadPayload) {
 			return ErrResp(http.StatusBadRequest, err, "")

pkg/services/ngalert/api/api_alertmanager_test.go

@@ -617,7 +617,7 @@ func TestRouteCreateSilence(t *testing.T) {
 				alertmanagerFor, err := sut.mam.AlertmanagerFor(1)
 				require.NoError(t, err)
 				silence.ID = ""
-				newID, err := alertmanagerFor.CreateSilence(&silence)
+				newID, err := alertmanagerFor.CreateSilence(context.Background(), &silence)
 				require.NoError(t, err)
 				silence.ID = newID
 			}

pkg/services/ngalert/notifier/external_alertmanager.go

@@ -2,15 +2,17 @@ package notifier
 
 import (
 	"context"
-	"errors"
+	"fmt"
 	"net/http"
 	"net/url"
 
 	httptransport "github.com/go-openapi/runtime/client"
+	"github.com/go-openapi/strfmt"
 	"github.com/grafana/grafana/pkg/infra/log"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	amclient "github.com/prometheus/alertmanager/api/v2/client"
+	amsilence "github.com/prometheus/alertmanager/api/v2/client/silence"
 )
 
 type externalAlertmanager struct {
@@ -40,15 +42,16 @@ func newExternalAlertmanager(cfg externalAlertmanagerConfig, orgID int64) (*exte
 	}
 
 	if cfg.URL == "" {
-		return nil, errors.New("empty URL")
+		return nil, fmt.Errorf("empty URL for tenant %s", cfg.TenantID)
 	}
 
 	u, err := url.Parse(cfg.URL)
 	if err != nil {
 		return nil, err
 	}
+	u = u.JoinPath(amclient.DefaultBasePath)
 
-	transport := httptransport.NewWithClient(u.Host, amclient.DefaultBasePath, []string{u.Scheme}, &client)
+	transport := httptransport.NewWithClient(u.Host, u.Path, []string{u.Scheme}, &client)
 
 	_, err = Load([]byte(cfg.DefaultConfig))
 	if err != nil {
@@ -74,24 +77,52 @@ func (am *externalAlertmanager) SaveAndApplyDefaultConfig(ctx context.Context) e
 	return nil
 }
 
-func (am *externalAlertmanager) GetStatus() (apimodels.GettableStatus, error) {
-	return apimodels.GettableStatus{}, nil
+func (am *externalAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) {
+	params := amsilence.NewPostSilencesParamsWithContext(ctx).WithSilence(silence)
+	res, err := am.amClient.Silence.PostSilences(params)
+	if err != nil {
+		return "", err
+	}
+
+	return res.Payload.SilenceID, nil
 }
 
-func (am *externalAlertmanager) CreateSilence(*apimodels.PostableSilence) (string, error) {
-	return "", nil
-}
-
-func (am *externalAlertmanager) DeleteSilence(string) error {
+func (am *externalAlertmanager) DeleteSilence(ctx context.Context, silenceID string) error {
+	params := amsilence.NewDeleteSilenceParamsWithContext(ctx).WithSilenceID(strfmt.UUID(silenceID))
+	_, err := am.amClient.Silence.DeleteSilence(params)
+	if err != nil {
+		return err
+	}
 	return nil
 }
 
-func (am *externalAlertmanager) GetSilence(silenceID string) (apimodels.GettableSilence, error) {
-	return apimodels.GettableSilence{}, nil
+func (am *externalAlertmanager) GetSilence(ctx context.Context, silenceID string) (apimodels.GettableSilence, error) {
+	params := amsilence.NewGetSilenceParamsWithContext(ctx).WithSilenceID(strfmt.UUID(silenceID))
+	res, err := am.amClient.Silence.GetSilence(params)
+	if err != nil {
+		return apimodels.GettableSilence{}, err
+	}
+
+	if res != nil {
+		return *res.Payload, nil
+	}
+
+	// In theory, this should never happen as is not possible for GetSilence to return an empty payload but no error.
+	return apimodels.GettableSilence{}, fmt.Errorf("unexpected error while trying to fetch silence: %s", silenceID)
 }
 
-func (am *externalAlertmanager) ListSilences([]string) (apimodels.GettableSilences, error) {
-	return apimodels.GettableSilences{}, nil
+func (am *externalAlertmanager) ListSilences(ctx context.Context, filter []string) (apimodels.GettableSilences, error) {
+	params := amsilence.NewGetSilencesParamsWithContext(ctx).WithFilter(filter)
+	res, err := am.amClient.Silence.GetSilences(params)
+	if err != nil {
+		return apimodels.GettableSilences{}, err
+	}
+
+	return res.Payload, nil
+}
+
+func (am *externalAlertmanager) GetStatus() apimodels.GettableStatus {
+	return apimodels.GettableStatus{}
 }
 
 func (am *externalAlertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) {
@@ -106,8 +137,8 @@ func (am *externalAlertmanager) PutAlerts(postableAlerts apimodels.PostableAlert
 	return nil
 }
 
-func (am *externalAlertmanager) GetReceivers(ctx context.Context) ([]apimodels.Receiver, error) {
-	return []apimodels.Receiver{}, nil
+func (am *externalAlertmanager) GetReceivers(ctx context.Context) []apimodels.Receiver {
+	return []apimodels.Receiver{}
 }
 
 func (am *externalAlertmanager) ApplyConfig(ctx context.Context, config *models.AlertConfiguration) error {

pkg/services/ngalert/notifier/external_alertmanager_test.go

@@ -1,13 +1,19 @@
 package notifier
 
 import (
+	"context"
 	"testing"
 
+	"github.com/go-openapi/strfmt"
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	amfake "github.com/grafana/grafana/pkg/services/ngalert/notifier/fake"
+	amv2 "github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/stretchr/testify/require"
 )
 
+const validConfig = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}`
+
 func TestNewExternalAlertmanager(t *testing.T) {
-	validConfig := `{"template_files":null,"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"email receiver","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}`
 	tests := []struct {
 		name          string
 		url           string
@@ -24,7 +30,7 @@ func TestNewExternalAlertmanager(t *testing.T) {
 			password:      "test",
 			defaultConfig: validConfig,
 			orgID:         1,
-			expErr:        "empty URL",
+			expErr:        "empty URL for tenant 1234",
 		},
 		{
 			name:          "empty default config",
@@ -78,3 +84,89 @@ func TestNewExternalAlertmanager(t *testing.T) {
 		})
 	}
 }
+
+func TestSilences(t *testing.T) {
+	const (
+		tenantID = "1"
+		password = "password"
+	)
+	fakeAm := amfake.NewFakeExternalAlertmanager(t, tenantID, password)
+
+	// Using a wrong password should cause an error.
+	cfg := externalAlertmanagerConfig{
+		URL:               fakeAm.Server.URL + "/alertmanager",
+		TenantID:          tenantID,
+		BasicAuthPassword: "wrongpassword",
+		DefaultConfig:     validConfig,
+	}
+	am, err := newExternalAlertmanager(cfg, 1)
+	require.NoError(t, err)
+
+	_, err = am.ListSilences(context.Background(), []string{})
+	require.NotNil(t, err)
+
+	// Using the correct password should make the request succeed.
+	cfg.BasicAuthPassword = password
+	am, err = newExternalAlertmanager(cfg, 1)
+	require.NoError(t, err)
+
+	// We should have no silences at first.
+	silences, err := am.ListSilences(context.Background(), []string{})
+	require.NoError(t, err)
+	require.Equal(t, 0, len(silences))
+
+	// Creating a silence should succeed.
+	testSilence := createSilence("test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime())
+	silenceID, err := am.CreateSilence(context.Background(), &testSilence)
+	require.NoError(t, err)
+	require.NotEmpty(t, silenceID)
+
+	// We should be able to retrieve a specific silence.
+	silence, err := am.GetSilence(context.Background(), silenceID)
+	require.NoError(t, err)
+	require.Equal(t, *testSilence.Comment, *silence.Comment)
+	require.Equal(t, *testSilence.CreatedBy, *silence.CreatedBy)
+	require.Equal(t, *testSilence.StartsAt, *silence.StartsAt)
+	require.Equal(t, *testSilence.EndsAt, *silence.EndsAt)
+	require.Equal(t, testSilence.Matchers, silence.Matchers)
+
+	// Trying to retrieve a non-existing silence should fail.
+	_, err = am.GetSilence(context.Background(), "invalid")
+	require.Error(t, err)
+
+	// After creating another silence, the total amount should be 2.
+	testSilence2 := createSilence("another test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime())
+	silenceID2, err := am.CreateSilence(context.Background(), &testSilence2)
+	require.NoError(t, err)
+	require.NotEmpty(t, silenceID2)
+
+	silences, err = am.ListSilences(context.Background(), []string{})
+	require.NoError(t, err)
+	require.Equal(t, 2, len(silences))
+	require.True(t, *silences[0].ID == silenceID || *silences[0].ID == silenceID2)
+	require.True(t, *silences[1].ID == silenceID || *silences[1].ID == silenceID2)
+
+	// After deleting one of those silences, the total amount should be 2.
+	err = am.DeleteSilence(context.Background(), silenceID)
+	require.NoError(t, err)
+
+	silences, err = am.ListSilences(context.Background(), []string{})
+	require.NoError(t, err)
+	require.Equal(t, 1, len(silences))
+
+	// Trying to delete the same error should fail.
+	err = am.DeleteSilence(context.Background(), silenceID)
+	require.NotNil(t, err)
+}
+
+func createSilence(comment, createdBy string, matchers amv2.Matchers, startsAt, endsAt strfmt.DateTime) apimodels.PostableSilence {
+	return apimodels.PostableSilence{
+		Silence: amv2.Silence{
+			Comment:   &comment,
+			CreatedBy: &createdBy,
+			Matchers:  matchers,
+			StartsAt:  &startsAt,
+			EndsAt:    &endsAt,
+		},
+	}
+}

pkg/services/ngalert/notifier/fake/external_alertmanager_fake.go

@@ -0,0 +1,154 @@
+package fake
+
+import (
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+
+	"github.com/go-openapi/strfmt"
+	alertingNotify "github.com/grafana/alerting/notify"
+	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/grafana/grafana/pkg/util"
+	"github.com/grafana/grafana/pkg/web"
+	"github.com/stretchr/testify/require"
+)
+
+type FakeExternalAlertmanager struct {
+	t        *testing.T
+	mtx      sync.RWMutex
+	tenantID string
+	password string
+	Server   *httptest.Server
+
+	silences alertingNotify.GettableSilences
+}
+
+func NewFakeExternalAlertmanager(t *testing.T, tenantID, password string) *FakeExternalAlertmanager {
+	t.Helper()
+	am := &FakeExternalAlertmanager{
+		t:        t,
+		tenantID: tenantID,
+		password: password,
+		mtx:      sync.RWMutex{},
+	}
+	mux := web.New()
+	mux.SetURLPrefix("/alertmanager/api/")
+	mux.UseMiddleware(am.basicAuthMiddleware)
+	mux.UseMiddleware(am.contentTypeJSONMiddleware)
+
+	// Routes
+	mux.Get("/v2/silences", http.HandlerFunc(am.getSilences))
+	mux.Get("/v2/silence/:silenceID", http.HandlerFunc(am.getSilence))
+	mux.Post("/v2/silences", http.HandlerFunc(am.postSilence))
+	mux.Delete("/v2/silence/:silenceID", http.HandlerFunc(am.deleteSilence))
+
+	am.Server = httptest.NewServer(mux)
+	return am
+}
+
+func (am *FakeExternalAlertmanager) getSilences(w http.ResponseWriter, r *http.Request) {
+	am.mtx.RLock()
+	if err := json.NewEncoder(w).Encode(am.silences); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+	am.mtx.RUnlock()
+}
+
+func (am *FakeExternalAlertmanager) getSilence(w http.ResponseWriter, r *http.Request) {
+	silenceID, ok := web.Params(r)[":silenceID"]
+	if !ok {
+		return
+	}
+
+	am.mtx.RLock()
+	var matching *alertingNotify.GettableSilence
+	for _, silence := range am.silences {
+		if *silence.ID == silenceID {
+			matching = silence
+			break
+		}
+	}
+	am.mtx.RUnlock()
+
+	if matching == nil {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(matching); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func (am *FakeExternalAlertmanager) postSilence(w http.ResponseWriter, r *http.Request) {
+	var silence definitions.PostableSilence
+	require.NoError(am.t, json.NewDecoder(r.Body).Decode(&silence))
+
+	updatedAt := strfmt.NewDateTime()
+	id := util.GenerateShortUID()
+
+	am.mtx.Lock()
+	am.silences = append(am.silences, &alertingNotify.GettableSilence{
+		ID:        &id,
+		UpdatedAt: &updatedAt,
+		Silence:   silence.Silence,
+	})
+	am.mtx.Unlock()
+
+	res := map[string]string{"silenceID": id}
+	if err := json.NewEncoder(w).Encode(res); err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+	}
+}
+
+func (am *FakeExternalAlertmanager) deleteSilence(w http.ResponseWriter, r *http.Request) {
+	silenceID, ok := web.Params(r)[":silenceID"]
+	if !ok {
+		return
+	}
+
+	am.mtx.Lock()
+	defer am.mtx.Unlock()
+	var newSilences []*alertingNotify.GettableSilence
+	for _, silence := range am.silences {
+		if *silence.ID != silenceID {
+			newSilences = append(newSilences, silence)
+		}
+	}
+
+	if len(newSilences) == len(am.silences) {
+		w.WriteHeader(http.StatusNotFound)
+		return
+	}
+
+	am.silences = newSilences
+	w.WriteHeader(http.StatusOK)
+}
+
+func (am *FakeExternalAlertmanager) basicAuthMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		username, password, ok := r.BasicAuth()
+		if !ok {
+			w.WriteHeader(http.StatusUnauthorized)
+			return
+		}
+		if username != am.tenantID || password != am.password || r.Header.Get("X-Scope-OrgID") != am.tenantID {
+			w.WriteHeader(http.StatusForbidden)
+			return
+		}
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (am *FakeExternalAlertmanager) contentTypeJSONMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		next.ServeHTTP(w, r)
+	})
+}
+
+func (am *FakeExternalAlertmanager) Close() {
+	am.Server.Close()
+}

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -35,12 +35,13 @@ type Alertmanager interface {
 	SaveAndApplyConfig(ctx context.Context, config *apimodels.PostableUserConfig) error
 	SaveAndApplyDefaultConfig(ctx context.Context) error
 	GetStatus() apimodels.GettableStatus
+	ApplyConfig(context.Context, *models.AlertConfiguration) error
 
 	// Silences
-	CreateSilence(*apimodels.PostableSilence) (string, error)
-	DeleteSilence(string) error
-	GetSilence(string) (apimodels.GettableSilence, error)
-	ListSilences([]string) (apimodels.GettableSilences, error)
+	CreateSilence(context.Context, *apimodels.PostableSilence) (string, error)
+	DeleteSilence(context.Context, string) error
+	GetSilence(context.Context, string) (apimodels.GettableSilence, error)
+	ListSilences(context.Context, []string) (apimodels.GettableSilences, error)
 
 	// Alerts
 	GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error)
@@ -51,7 +52,6 @@ type Alertmanager interface {
 	GetReceivers(ctx context.Context) []apimodels.Receiver
 	TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*TestReceiversResult, error)
 	TestTemplate(ctx context.Context, c apimodels.TestTemplatesConfigBodyParams) (*TestTemplatesResults, error)
-	ApplyConfig(context.Context, *models.AlertConfiguration) error
 
 	// State
 	StopAndWait()

pkg/services/ngalert/notifier/silences.go

@@ -1,21 +1,23 @@
 package notifier
 
 import (
+	"context"
+
 	alertingNotify "github.com/grafana/alerting/notify"
 )
 
-func (am *alertmanager) ListSilences(filter []string) (alertingNotify.GettableSilences, error) {
+func (am *alertmanager) ListSilences(_ context.Context, filter []string) (alertingNotify.GettableSilences, error) {
 	return am.Base.ListSilences(filter)
 }
 
-func (am *alertmanager) GetSilence(silenceID string) (alertingNotify.GettableSilence, error) {
+func (am *alertmanager) GetSilence(_ context.Context, silenceID string) (alertingNotify.GettableSilence, error) {
 	return am.Base.GetSilence(silenceID)
 }
 
-func (am *alertmanager) CreateSilence(ps *alertingNotify.PostableSilence) (string, error) {
+func (am *alertmanager) CreateSilence(_ context.Context, ps *alertingNotify.PostableSilence) (string, error) {
 	return am.Base.CreateSilence(ps)
 }
 
-func (am *alertmanager) DeleteSilence(silenceID string) error {
+func (am *alertmanager) DeleteSilence(_ context.Context, silenceID string) error {
 	return am.Base.DeleteSilence(silenceID)
 }