mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Use Mimir image to run integration tests for the remote Alertmanager (#76608)
* Alerting: Use Mimir image to run integration tests for the remote Alertmanager * skip integration test when running all tests * skipping integration test when no Alertmanager URL is provided * fix bad host for mimir_backend * remove basic auth testing until we have an nginx image in our CI
This commit is contained in:
Santiago
GitHub
@@ -2,11 +2,14 @@ package notifier
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/rand"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-openapi/strfmt"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
amfake "github.com/grafana/grafana/pkg/services/ngalert/notifier/fake"
|
||||
"github.com/grafana/grafana/pkg/util"
|
||||
amv2 "github.com/prometheus/alertmanager/api/v2/models"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
@@ -85,88 +88,103 @@ func TestNewExternalAlertmanager(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestSilences(t *testing.T) {
|
||||
const (
|
||||
tenantID = "1"
|
||||
password = "password"
|
||||
)
|
||||
fakeAm := amfake.NewFakeExternalAlertmanager(t, tenantID, password)
|
||||
func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skipping integration test")
|
||||
}
|
||||
|
||||
amURL, ok := os.LookupEnv("AM_URL")
|
||||
if !ok {
|
||||
t.Skip("No Alertmanager URL provided")
|
||||
}
|
||||
tenantID := os.Getenv("AM_TENANT_ID")
|
||||
password := os.Getenv("AM_PASSWORD")
|
||||
|
||||
// Using a wrong password should cause an error.
|
||||
cfg := externalAlertmanagerConfig{
|
||||
URL: fakeAm.Server.URL + "/alertmanager",
|
||||
URL: amURL + "/alertmanager",
|
||||
TenantID: tenantID,
|
||||
BasicAuthPassword: "wrongpassword",
|
||||
BasicAuthPassword: password,
|
||||
DefaultConfig: validConfig,
|
||||
}
|
||||
am, err := newExternalAlertmanager(cfg, 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = am.ListSilences(context.Background(), []string{})
|
||||
require.NotNil(t, err)
|
||||
|
||||
// Using the correct password should make the request succeed.
|
||||
cfg.BasicAuthPassword = password
|
||||
am, err = newExternalAlertmanager(cfg, 1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// We should have no silences at first.
|
||||
silences, err := am.ListSilences(context.Background(), []string{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 0, len(silences))
|
||||
|
||||
// Creating a silence should succeed.
|
||||
testSilence := createSilence("test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime())
|
||||
silenceID, err := am.CreateSilence(context.Background(), &testSilence)
|
||||
testSilence := genSilence("test")
|
||||
id, err := am.CreateSilence(context.Background(), &testSilence)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, silenceID)
|
||||
require.NotEmpty(t, id)
|
||||
testSilence.ID = id
|
||||
|
||||
// We should be able to retrieve a specific silence.
|
||||
silence, err := am.GetSilence(context.Background(), silenceID)
|
||||
silence, err := am.GetSilence(context.Background(), testSilence.ID)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, *testSilence.Comment, *silence.Comment)
|
||||
require.Equal(t, *testSilence.CreatedBy, *silence.CreatedBy)
|
||||
require.Equal(t, *testSilence.StartsAt, *silence.StartsAt)
|
||||
require.Equal(t, *testSilence.EndsAt, *silence.EndsAt)
|
||||
require.Equal(t, testSilence.Matchers, silence.Matchers)
|
||||
require.Equal(t, testSilence.ID, *silence.ID)
|
||||
|
||||
// Trying to retrieve a non-existing silence should fail.
|
||||
_, err = am.GetSilence(context.Background(), "invalid")
|
||||
_, err = am.GetSilence(context.Background(), util.GenerateShortUID())
|
||||
require.Error(t, err)
|
||||
|
||||
// After creating another silence, the total amount should be 2.
|
||||
testSilence2 := createSilence("another test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime())
|
||||
silenceID2, err := am.CreateSilence(context.Background(), &testSilence2)
|
||||
testSilence2 := genSilence("test")
|
||||
id, err = am.CreateSilence(context.Background(), &testSilence2)
|
||||
require.NoError(t, err)
|
||||
require.NotEmpty(t, silenceID2)
|
||||
require.NotEmpty(t, id)
|
||||
testSilence2.ID = id
|
||||
|
||||
silences, err = am.ListSilences(context.Background(), []string{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 2, len(silences))
|
||||
require.True(t, *silences[0].ID == silenceID || *silences[0].ID == silenceID2)
|
||||
require.True(t, *silences[1].ID == silenceID || *silences[1].ID == silenceID2)
|
||||
require.True(t, *silences[0].ID == testSilence.ID || *silences[0].ID == testSilence2.ID)
|
||||
require.True(t, *silences[1].ID == testSilence.ID || *silences[1].ID == testSilence2.ID)
|
||||
|
||||
// After deleting one of those silences, the total amount should be 2.
|
||||
err = am.DeleteSilence(context.Background(), silenceID)
|
||||
// After deleting one of those silences, the total amount should be 2 but one of those should be expired.
|
||||
err = am.DeleteSilence(context.Background(), testSilence.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
silences, err = am.ListSilences(context.Background(), []string{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, 1, len(silences))
|
||||
|
||||
// Trying to delete the same error should fail.
|
||||
err = am.DeleteSilence(context.Background(), silenceID)
|
||||
require.NotNil(t, err)
|
||||
for _, s := range silences {
|
||||
if *s.ID == testSilence.ID {
|
||||
require.Equal(t, *s.Status.State, "expired")
|
||||
} else {
|
||||
require.Equal(t, *s.Status.State, "pending")
|
||||
}
|
||||
}
|
||||
|
||||
// When deleting the other silence, both should be expired.
|
||||
err = am.DeleteSilence(context.Background(), testSilence2.ID)
|
||||
require.NoError(t, err)
|
||||
|
||||
silences, err = am.ListSilences(context.Background(), []string{})
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, *silences[0].Status.State, "expired")
|
||||
require.Equal(t, *silences[1].Status.State, "expired")
|
||||
}
|
||||
|
||||
func createSilence(comment, createdBy string, matchers amv2.Matchers, startsAt, endsAt strfmt.DateTime) apimodels.PostableSilence {
|
||||
func genSilence(createdBy string) apimodels.PostableSilence {
|
||||
starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second))
|
||||
ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second))
|
||||
comment := "test comment"
|
||||
isEqual := true
|
||||
name := "test"
|
||||
value := "test"
|
||||
isRegex := false
|
||||
matchers := amv2.Matchers{&amv2.Matcher{IsEqual: &isEqual, Name: &name, Value: &value, IsRegex: &isRegex}}
|
||||
|
||||
return apimodels.PostableSilence{
|
||||
Silence: amv2.Silence{
|
||||
Comment: &comment,
|
||||
CreatedBy: &createdBy,
|
||||
Matchers: matchers,
|
||||
StartsAt: &startsAt,
|
||||
EndsAt: &endsAt,
|
||||
StartsAt: &starts,
|
||||
EndsAt: &ends,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,154 +0,0 @@
|
||||
package fake
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/go-openapi/strfmt"
|
||||
alertingNotify "github.com/grafana/alerting/notify"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/grafana/grafana/pkg/util"
|
||||
"github.com/grafana/grafana/pkg/web"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
type FakeExternalAlertmanager struct {
|
||||
t *testing.T
|
||||
mtx sync.RWMutex
|
||||
tenantID string
|
||||
password string
|
||||
Server *httptest.Server
|
||||
|
||||
silences alertingNotify.GettableSilences
|
||||
}
|
||||
|
||||
func NewFakeExternalAlertmanager(t *testing.T, tenantID, password string) *FakeExternalAlertmanager {
|
||||
t.Helper()
|
||||
am := &FakeExternalAlertmanager{
|
||||
t: t,
|
||||
tenantID: tenantID,
|
||||
password: password,
|
||||
mtx: sync.RWMutex{},
|
||||
}
|
||||
mux := web.New()
|
||||
mux.SetURLPrefix("/alertmanager/api/")
|
||||
mux.UseMiddleware(am.basicAuthMiddleware)
|
||||
mux.UseMiddleware(am.contentTypeJSONMiddleware)
|
||||
|
||||
// Routes
|
||||
mux.Get("/v2/silences", http.HandlerFunc(am.getSilences))
|
||||
mux.Get("/v2/silence/:silenceID", http.HandlerFunc(am.getSilence))
|
||||
mux.Post("/v2/silences", http.HandlerFunc(am.postSilence))
|
||||
mux.Delete("/v2/silence/:silenceID", http.HandlerFunc(am.deleteSilence))
|
||||
|
||||
am.Server = httptest.NewServer(mux)
|
||||
return am
|
||||
}
|
||||
|
||||
func (am *FakeExternalAlertmanager) getSilences(w http.ResponseWriter, r *http.Request) {
|
||||
am.mtx.RLock()
|
||||
if err := json.NewEncoder(w).Encode(am.silences); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
am.mtx.RUnlock()
|
||||
}
|
||||
|
||||
func (am *FakeExternalAlertmanager) getSilence(w http.ResponseWriter, r *http.Request) {
|
||||
silenceID, ok := web.Params(r)[":silenceID"]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
am.mtx.RLock()
|
||||
var matching *alertingNotify.GettableSilence
|
||||
for _, silence := range am.silences {
|
||||
if *silence.ID == silenceID {
|
||||
matching = silence
|
||||
break
|
||||
}
|
||||
}
|
||||
am.mtx.RUnlock()
|
||||
|
||||
if matching == nil {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
if err := json.NewEncoder(w).Encode(matching); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
func (am *FakeExternalAlertmanager) postSilence(w http.ResponseWriter, r *http.Request) {
|
||||
var silence definitions.PostableSilence
|
||||
require.NoError(am.t, json.NewDecoder(r.Body).Decode(&silence))
|
||||
|
||||
updatedAt := strfmt.NewDateTime()
|
||||
id := util.GenerateShortUID()
|
||||
|
||||
am.mtx.Lock()
|
||||
am.silences = append(am.silences, &alertingNotify.GettableSilence{
|
||||
ID: &id,
|
||||
UpdatedAt: &updatedAt,
|
||||
Silence: silence.Silence,
|
||||
})
|
||||
am.mtx.Unlock()
|
||||
|
||||
res := map[string]string{"silenceID": id}
|
||||
if err := json.NewEncoder(w).Encode(res); err != nil {
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}
|
||||
}
|
||||
|
||||
func (am *FakeExternalAlertmanager) deleteSilence(w http.ResponseWriter, r *http.Request) {
|
||||
silenceID, ok := web.Params(r)[":silenceID"]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
|
||||
am.mtx.Lock()
|
||||
defer am.mtx.Unlock()
|
||||
var newSilences []*alertingNotify.GettableSilence
|
||||
for _, silence := range am.silences {
|
||||
if *silence.ID != silenceID {
|
||||
newSilences = append(newSilences, silence)
|
||||
}
|
||||
}
|
||||
|
||||
if len(newSilences) == len(am.silences) {
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
am.silences = newSilences
|
||||
w.WriteHeader(http.StatusOK)
|
||||
}
|
||||
|
||||
func (am *FakeExternalAlertmanager) basicAuthMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
username, password, ok := r.BasicAuth()
|
||||
if !ok {
|
||||
w.WriteHeader(http.StatusUnauthorized)
|
||||
return
|
||||
}
|
||||
if username != am.tenantID || password != am.password || r.Header.Get("X-Scope-OrgID") != am.tenantID {
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (am *FakeExternalAlertmanager) contentTypeJSONMiddleware(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (am *FakeExternalAlertmanager) Close() {
|
||||
am.Server.Close()
|
||||
}
|
||||
Reference in New Issue
Block a user