diff --git a/pkg/services/ngalert/api/api_alertmanager.go b/pkg/services/ngalert/api/api_alertmanager.go index 9b680b6ba21..da1dc7d09c1 100644 --- a/pkg/services/ngalert/api/api_alertmanager.go +++ b/pkg/services/ngalert/api/api_alertmanager.go @@ -75,7 +75,7 @@ func (srv AlertmanagerSrv) RouteCreateSilence(c *contextmodel.ReqContext, postab return ErrResp(http.StatusUnauthorized, fmt.Errorf("user is not authorized to %s silences", errAction), "") } - silenceID, err := am.CreateSilence(&postableSilence) + silenceID, err := am.CreateSilence(c.Req.Context(), &postableSilence) if err != nil { if errors.Is(err, alertingNotify.ErrSilenceNotFound) { return ErrResp(http.StatusNotFound, err, "") @@ -112,7 +112,7 @@ func (srv AlertmanagerSrv) RouteDeleteSilence(c *contextmodel.ReqContext, silenc return errResp } - if err := am.DeleteSilence(silenceID); err != nil { + if err := am.DeleteSilence(c.Req.Context(), silenceID); err != nil { if errors.Is(err, alertingNotify.ErrSilenceNotFound) { return ErrResp(http.StatusNotFound, err, "") } @@ -199,7 +199,7 @@ func (srv AlertmanagerSrv) RouteGetSilence(c *contextmodel.ReqContext, silenceID return errResp } - gettableSilence, err := am.GetSilence(silenceID) + gettableSilence, err := am.GetSilence(c.Req.Context(), silenceID) if err != nil { if errors.Is(err, alertingNotify.ErrSilenceNotFound) { return ErrResp(http.StatusNotFound, err, "") @@ -216,7 +216,7 @@ func (srv AlertmanagerSrv) RouteGetSilences(c *contextmodel.ReqContext) response return errResp } - gettableSilences, err := am.ListSilences(c.QueryStrings("filter")) + gettableSilences, err := am.ListSilences(c.Req.Context(), c.QueryStrings("filter")) if err != nil { if errors.Is(err, alertingNotify.ErrListSilencesBadPayload) { return ErrResp(http.StatusBadRequest, err, "") diff --git a/pkg/services/ngalert/api/api_alertmanager_test.go b/pkg/services/ngalert/api/api_alertmanager_test.go index e27caf0e5fe..cef372a4dc9 100644 --- a/pkg/services/ngalert/api/api_alertmanager_test.go +++ b/pkg/services/ngalert/api/api_alertmanager_test.go @@ -617,7 +617,7 @@ func TestRouteCreateSilence(t *testing.T) { alertmanagerFor, err := sut.mam.AlertmanagerFor(1) require.NoError(t, err) silence.ID = "" - newID, err := alertmanagerFor.CreateSilence(&silence) + newID, err := alertmanagerFor.CreateSilence(context.Background(), &silence) require.NoError(t, err) silence.ID = newID } diff --git a/pkg/services/ngalert/notifier/external_alertmanager.go b/pkg/services/ngalert/notifier/external_alertmanager.go index 7b24323b21f..bc8e475b64e 100644 --- a/pkg/services/ngalert/notifier/external_alertmanager.go +++ b/pkg/services/ngalert/notifier/external_alertmanager.go @@ -2,15 +2,17 @@ package notifier import ( "context" - "errors" + "fmt" "net/http" "net/url" httptransport "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" "github.com/grafana/grafana/pkg/infra/log" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/models" amclient "github.com/prometheus/alertmanager/api/v2/client" + amsilence "github.com/prometheus/alertmanager/api/v2/client/silence" ) type externalAlertmanager struct { @@ -40,15 +42,16 @@ func newExternalAlertmanager(cfg externalAlertmanagerConfig, orgID int64) (*exte } if cfg.URL == "" { - return nil, errors.New("empty URL") + return nil, fmt.Errorf("empty URL for tenant %s", cfg.TenantID) } u, err := url.Parse(cfg.URL) if err != nil { return nil, err } + u = u.JoinPath(amclient.DefaultBasePath) - transport := httptransport.NewWithClient(u.Host, amclient.DefaultBasePath, []string{u.Scheme}, &client) + transport := httptransport.NewWithClient(u.Host, u.Path, []string{u.Scheme}, &client) _, err = Load([]byte(cfg.DefaultConfig)) if err != nil { @@ -74,24 +77,52 @@ func (am *externalAlertmanager) SaveAndApplyDefaultConfig(ctx context.Context) e return nil } -func (am *externalAlertmanager) GetStatus() (apimodels.GettableStatus, error) { - return apimodels.GettableStatus{}, nil +func (am *externalAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) { + params := amsilence.NewPostSilencesParamsWithContext(ctx).WithSilence(silence) + res, err := am.amClient.Silence.PostSilences(params) + if err != nil { + return "", err + } + + return res.Payload.SilenceID, nil } -func (am *externalAlertmanager) CreateSilence(*apimodels.PostableSilence) (string, error) { - return "", nil -} - -func (am *externalAlertmanager) DeleteSilence(string) error { +func (am *externalAlertmanager) DeleteSilence(ctx context.Context, silenceID string) error { + params := amsilence.NewDeleteSilenceParamsWithContext(ctx).WithSilenceID(strfmt.UUID(silenceID)) + _, err := am.amClient.Silence.DeleteSilence(params) + if err != nil { + return err + } return nil } -func (am *externalAlertmanager) GetSilence(silenceID string) (apimodels.GettableSilence, error) { - return apimodels.GettableSilence{}, nil +func (am *externalAlertmanager) GetSilence(ctx context.Context, silenceID string) (apimodels.GettableSilence, error) { + params := amsilence.NewGetSilenceParamsWithContext(ctx).WithSilenceID(strfmt.UUID(silenceID)) + res, err := am.amClient.Silence.GetSilence(params) + if err != nil { + return apimodels.GettableSilence{}, err + } + + if res != nil { + return *res.Payload, nil + } + + // In theory, this should never happen as is not possible for GetSilence to return an empty payload but no error. + return apimodels.GettableSilence{}, fmt.Errorf("unexpected error while trying to fetch silence: %s", silenceID) } -func (am *externalAlertmanager) ListSilences([]string) (apimodels.GettableSilences, error) { - return apimodels.GettableSilences{}, nil +func (am *externalAlertmanager) ListSilences(ctx context.Context, filter []string) (apimodels.GettableSilences, error) { + params := amsilence.NewGetSilencesParamsWithContext(ctx).WithFilter(filter) + res, err := am.amClient.Silence.GetSilences(params) + if err != nil { + return apimodels.GettableSilences{}, err + } + + return res.Payload, nil +} + +func (am *externalAlertmanager) GetStatus() apimodels.GettableStatus { + return apimodels.GettableStatus{} } func (am *externalAlertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) { @@ -106,8 +137,8 @@ func (am *externalAlertmanager) PutAlerts(postableAlerts apimodels.PostableAlert return nil } -func (am *externalAlertmanager) GetReceivers(ctx context.Context) ([]apimodels.Receiver, error) { - return []apimodels.Receiver{}, nil +func (am *externalAlertmanager) GetReceivers(ctx context.Context) []apimodels.Receiver { + return []apimodels.Receiver{} } func (am *externalAlertmanager) ApplyConfig(ctx context.Context, config *models.AlertConfiguration) error { diff --git a/pkg/services/ngalert/notifier/external_alertmanager_test.go b/pkg/services/ngalert/notifier/external_alertmanager_test.go index 3304053d954..8985b0135e0 100644 --- a/pkg/services/ngalert/notifier/external_alertmanager_test.go +++ b/pkg/services/ngalert/notifier/external_alertmanager_test.go @@ -1,13 +1,19 @@ package notifier import ( + "context" "testing" + "github.com/go-openapi/strfmt" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + amfake "github.com/grafana/grafana/pkg/services/ngalert/notifier/fake" + amv2 "github.com/prometheus/alertmanager/api/v2/models" "github.com/stretchr/testify/require" ) +const validConfig = `{"template_files":{},"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"some other name","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}` + func TestNewExternalAlertmanager(t *testing.T) { - validConfig := `{"template_files":null,"alertmanager_config":{"route":{"receiver":"grafana-default-email","group_by":["grafana_folder","alertname"]},"templates":null,"receivers":[{"name":"grafana-default-email","grafana_managed_receiver_configs":[{"uid":"","name":"email receiver","type":"email","disableResolveMessage":false,"settings":{"addresses":"\u003cexample@email.com\u003e"},"secureSettings":null}]}]}}` tests := []struct { name string url string @@ -24,7 +30,7 @@ func TestNewExternalAlertmanager(t *testing.T) { password: "test", defaultConfig: validConfig, orgID: 1, - expErr: "empty URL", + expErr: "empty URL for tenant 1234", }, { name: "empty default config", @@ -78,3 +84,89 @@ func TestNewExternalAlertmanager(t *testing.T) { }) } } + +func TestSilences(t *testing.T) { + const ( + tenantID = "1" + password = "password" + ) + fakeAm := amfake.NewFakeExternalAlertmanager(t, tenantID, password) + + // Using a wrong password should cause an error. + cfg := externalAlertmanagerConfig{ + URL: fakeAm.Server.URL + "/alertmanager", + TenantID: tenantID, + BasicAuthPassword: "wrongpassword", + DefaultConfig: validConfig, + } + am, err := newExternalAlertmanager(cfg, 1) + require.NoError(t, err) + + _, err = am.ListSilences(context.Background(), []string{}) + require.NotNil(t, err) + + // Using the correct password should make the request succeed. + cfg.BasicAuthPassword = password + am, err = newExternalAlertmanager(cfg, 1) + require.NoError(t, err) + + // We should have no silences at first. + silences, err := am.ListSilences(context.Background(), []string{}) + require.NoError(t, err) + require.Equal(t, 0, len(silences)) + + // Creating a silence should succeed. + testSilence := createSilence("test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime()) + silenceID, err := am.CreateSilence(context.Background(), &testSilence) + require.NoError(t, err) + require.NotEmpty(t, silenceID) + + // We should be able to retrieve a specific silence. + silence, err := am.GetSilence(context.Background(), silenceID) + require.NoError(t, err) + require.Equal(t, *testSilence.Comment, *silence.Comment) + require.Equal(t, *testSilence.CreatedBy, *silence.CreatedBy) + require.Equal(t, *testSilence.StartsAt, *silence.StartsAt) + require.Equal(t, *testSilence.EndsAt, *silence.EndsAt) + require.Equal(t, testSilence.Matchers, silence.Matchers) + + // Trying to retrieve a non-existing silence should fail. + _, err = am.GetSilence(context.Background(), "invalid") + require.Error(t, err) + + // After creating another silence, the total amount should be 2. + testSilence2 := createSilence("another test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime()) + silenceID2, err := am.CreateSilence(context.Background(), &testSilence2) + require.NoError(t, err) + require.NotEmpty(t, silenceID2) + + silences, err = am.ListSilences(context.Background(), []string{}) + require.NoError(t, err) + require.Equal(t, 2, len(silences)) + require.True(t, *silences[0].ID == silenceID || *silences[0].ID == silenceID2) + require.True(t, *silences[1].ID == silenceID || *silences[1].ID == silenceID2) + + // After deleting one of those silences, the total amount should be 2. + err = am.DeleteSilence(context.Background(), silenceID) + require.NoError(t, err) + + silences, err = am.ListSilences(context.Background(), []string{}) + require.NoError(t, err) + require.Equal(t, 1, len(silences)) + + // Trying to delete the same error should fail. + err = am.DeleteSilence(context.Background(), silenceID) + require.NotNil(t, err) +} + +func createSilence(comment, createdBy string, matchers amv2.Matchers, startsAt, endsAt strfmt.DateTime) apimodels.PostableSilence { + return apimodels.PostableSilence{ + Silence: amv2.Silence{ + Comment: &comment, + CreatedBy: &createdBy, + Matchers: matchers, + StartsAt: &startsAt, + EndsAt: &endsAt, + }, + } +} diff --git a/pkg/services/ngalert/notifier/fake/external_alertmanager_fake.go b/pkg/services/ngalert/notifier/fake/external_alertmanager_fake.go new file mode 100644 index 00000000000..b32b0eae9e2 --- /dev/null +++ b/pkg/services/ngalert/notifier/fake/external_alertmanager_fake.go @@ -0,0 +1,154 @@ +package fake + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + "github.com/go-openapi/strfmt" + alertingNotify "github.com/grafana/alerting/notify" + "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + "github.com/grafana/grafana/pkg/util" + "github.com/grafana/grafana/pkg/web" + "github.com/stretchr/testify/require" +) + +type FakeExternalAlertmanager struct { + t *testing.T + mtx sync.RWMutex + tenantID string + password string + Server *httptest.Server + + silences alertingNotify.GettableSilences +} + +func NewFakeExternalAlertmanager(t *testing.T, tenantID, password string) *FakeExternalAlertmanager { + t.Helper() + am := &FakeExternalAlertmanager{ + t: t, + tenantID: tenantID, + password: password, + mtx: sync.RWMutex{}, + } + mux := web.New() + mux.SetURLPrefix("/alertmanager/api/") + mux.UseMiddleware(am.basicAuthMiddleware) + mux.UseMiddleware(am.contentTypeJSONMiddleware) + + // Routes + mux.Get("/v2/silences", http.HandlerFunc(am.getSilences)) + mux.Get("/v2/silence/:silenceID", http.HandlerFunc(am.getSilence)) + mux.Post("/v2/silences", http.HandlerFunc(am.postSilence)) + mux.Delete("/v2/silence/:silenceID", http.HandlerFunc(am.deleteSilence)) + + am.Server = httptest.NewServer(mux) + return am +} + +func (am *FakeExternalAlertmanager) getSilences(w http.ResponseWriter, r *http.Request) { + am.mtx.RLock() + if err := json.NewEncoder(w).Encode(am.silences); err != nil { + w.WriteHeader(http.StatusInternalServerError) + } + am.mtx.RUnlock() +} + +func (am *FakeExternalAlertmanager) getSilence(w http.ResponseWriter, r *http.Request) { + silenceID, ok := web.Params(r)[":silenceID"] + if !ok { + return + } + + am.mtx.RLock() + var matching *alertingNotify.GettableSilence + for _, silence := range am.silences { + if *silence.ID == silenceID { + matching = silence + break + } + } + am.mtx.RUnlock() + + if matching == nil { + w.WriteHeader(http.StatusNotFound) + return + } + + if err := json.NewEncoder(w).Encode(matching); err != nil { + w.WriteHeader(http.StatusInternalServerError) + } +} + +func (am *FakeExternalAlertmanager) postSilence(w http.ResponseWriter, r *http.Request) { + var silence definitions.PostableSilence + require.NoError(am.t, json.NewDecoder(r.Body).Decode(&silence)) + + updatedAt := strfmt.NewDateTime() + id := util.GenerateShortUID() + + am.mtx.Lock() + am.silences = append(am.silences, &alertingNotify.GettableSilence{ + ID: &id, + UpdatedAt: &updatedAt, + Silence: silence.Silence, + }) + am.mtx.Unlock() + + res := map[string]string{"silenceID": id} + if err := json.NewEncoder(w).Encode(res); err != nil { + w.WriteHeader(http.StatusInternalServerError) + } +} + +func (am *FakeExternalAlertmanager) deleteSilence(w http.ResponseWriter, r *http.Request) { + silenceID, ok := web.Params(r)[":silenceID"] + if !ok { + return + } + + am.mtx.Lock() + defer am.mtx.Unlock() + var newSilences []*alertingNotify.GettableSilence + for _, silence := range am.silences { + if *silence.ID != silenceID { + newSilences = append(newSilences, silence) + } + } + + if len(newSilences) == len(am.silences) { + w.WriteHeader(http.StatusNotFound) + return + } + + am.silences = newSilences + w.WriteHeader(http.StatusOK) +} + +func (am *FakeExternalAlertmanager) basicAuthMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + if !ok { + w.WriteHeader(http.StatusUnauthorized) + return + } + if username != am.tenantID || password != am.password || r.Header.Get("X-Scope-OrgID") != am.tenantID { + w.WriteHeader(http.StatusForbidden) + return + } + next.ServeHTTP(w, r) + }) +} + +func (am *FakeExternalAlertmanager) contentTypeJSONMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + next.ServeHTTP(w, r) + }) +} + +func (am *FakeExternalAlertmanager) Close() { + am.Server.Close() +} diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go index d4a052dc35e..edf286d3093 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go @@ -35,12 +35,13 @@ type Alertmanager interface { SaveAndApplyConfig(ctx context.Context, config *apimodels.PostableUserConfig) error SaveAndApplyDefaultConfig(ctx context.Context) error GetStatus() apimodels.GettableStatus + ApplyConfig(context.Context, *models.AlertConfiguration) error // Silences - CreateSilence(*apimodels.PostableSilence) (string, error) - DeleteSilence(string) error - GetSilence(string) (apimodels.GettableSilence, error) - ListSilences([]string) (apimodels.GettableSilences, error) + CreateSilence(context.Context, *apimodels.PostableSilence) (string, error) + DeleteSilence(context.Context, string) error + GetSilence(context.Context, string) (apimodels.GettableSilence, error) + ListSilences(context.Context, []string) (apimodels.GettableSilences, error) // Alerts GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) @@ -51,7 +52,6 @@ type Alertmanager interface { GetReceivers(ctx context.Context) []apimodels.Receiver TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*TestReceiversResult, error) TestTemplate(ctx context.Context, c apimodels.TestTemplatesConfigBodyParams) (*TestTemplatesResults, error) - ApplyConfig(context.Context, *models.AlertConfiguration) error // State StopAndWait() diff --git a/pkg/services/ngalert/notifier/silences.go b/pkg/services/ngalert/notifier/silences.go index de1acb4a533..035b4fca9f4 100644 --- a/pkg/services/ngalert/notifier/silences.go +++ b/pkg/services/ngalert/notifier/silences.go @@ -1,21 +1,23 @@ package notifier import ( + "context" + alertingNotify "github.com/grafana/alerting/notify" ) -func (am *alertmanager) ListSilences(filter []string) (alertingNotify.GettableSilences, error) { +func (am *alertmanager) ListSilences(_ context.Context, filter []string) (alertingNotify.GettableSilences, error) { return am.Base.ListSilences(filter) } -func (am *alertmanager) GetSilence(silenceID string) (alertingNotify.GettableSilence, error) { +func (am *alertmanager) GetSilence(_ context.Context, silenceID string) (alertingNotify.GettableSilence, error) { return am.Base.GetSilence(silenceID) } -func (am *alertmanager) CreateSilence(ps *alertingNotify.PostableSilence) (string, error) { +func (am *alertmanager) CreateSilence(_ context.Context, ps *alertingNotify.PostableSilence) (string, error) { return am.Base.CreateSilence(ps) } -func (am *alertmanager) DeleteSilence(silenceID string) error { +func (am *alertmanager) DeleteSilence(_ context.Context, silenceID string) error { return am.Base.DeleteSilence(silenceID) }