Alerting: Use Mimir image to run integration tests for the remote Alertmanager (#76608)

* Alerting: Use Mimir image to run integration tests for the remote Alertmanager

* skip integration test when running all tests

* skipping integration test when no Alertmanager URL is provided

* fix bad host for mimir_backend

* remove basic auth testing until we have an nginx image in our CI

.drone.yml

@@ -800,7 +800,7 @@ services:
   - /bin/mimir -target=backend
   environment: {}
   image: grafana/mimir:latest
-  name: mimir
+  name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
   name: redis
@@ -966,16 +966,20 @@ steps:
   image: golang:1.20.10-alpine
   name: memcached-integration-tests
 - commands:
-  - dockerize -wait tcp://mimir:8080 -timeout 120s
+  - dockerize -wait tcp://mimir_backend:8080 -timeout 120s
   image: jwilder/dockerize:0.6.1
   name: wait-for-remote-alertmanager
 - commands:
   - apk add --update build-base
   - go clean -testcache
-  - go test -run IntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/...
+  - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...
   depends_on:
   - wire-install
   - wait-for-remote-alertmanager
+  environment:
+    AM_PASSWORD: test
+    AM_TENANT_ID: test
+    AM_URL: http://mimir_backend:8080
   image: golang:1.20.10-alpine
   name: remote-alertmanager-integration-tests
 trigger:
@@ -1178,7 +1182,7 @@ services:
   - /bin/mimir -target=backend
   environment: {}
   image: grafana/mimir:latest
-  name: mimir
+  name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
   name: redis
@@ -2103,7 +2107,7 @@ services:
   - /bin/mimir -target=backend
   environment: {}
   image: grafana/mimir:latest
-  name: mimir
+  name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
   name: redis
@@ -2248,16 +2252,20 @@ steps:
   image: golang:1.20.10-alpine
   name: memcached-integration-tests
 - commands:
-  - dockerize -wait tcp://mimir:8080 -timeout 120s
+  - dockerize -wait tcp://mimir_backend:8080 -timeout 120s
   image: jwilder/dockerize:0.6.1
   name: wait-for-remote-alertmanager
 - commands:
   - apk add --update build-base
   - go clean -testcache
-  - go test -run IntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/...
+  - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...
   depends_on:
   - wire-install
   - wait-for-remote-alertmanager
+  environment:
+    AM_PASSWORD: test
+    AM_TENANT_ID: test
+    AM_URL: http://mimir_backend:8080
   image: golang:1.20.10-alpine
   name: remote-alertmanager-integration-tests
 trigger:
@@ -3762,7 +3770,7 @@ services:
   - /bin/mimir -target=backend
   environment: {}
   image: grafana/mimir:latest
-  name: mimir
+  name: mimir_backend
 - environment: {}
   image: redis:6.2.11-alpine
   name: redis
@@ -3900,16 +3908,20 @@ steps:
   image: golang:1.20.10-alpine
   name: memcached-integration-tests
 - commands:
-  - dockerize -wait tcp://mimir:8080 -timeout 120s
+  - dockerize -wait tcp://mimir_backend:8080 -timeout 120s
   image: jwilder/dockerize:0.6.1
   name: wait-for-remote-alertmanager
 - commands:
   - apk add --update build-base
   - go clean -testcache
-  - go test -run IntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/...
+  - go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...
   depends_on:
   - wire-install
   - wait-for-remote-alertmanager
+  environment:
+    AM_PASSWORD: test
+    AM_TENANT_ID: test
+    AM_URL: http://mimir_backend:8080
   image: golang:1.20.10-alpine
   name: remote-alertmanager-integration-tests
 trigger:
@@ -4598,6 +4610,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: ff105572d451a06880931bc5d3abdb86e50161eb84091e09118d9bf6a229e39b
+hmac: d6bd1e6c990959426e575500bd89b4e28cdbc991f245e0723dc912ccc4460470
 
 ...

Makefile

@@ -164,6 +164,13 @@ test-go-integration: ## Run integration tests for backend with flags.
 	@echo "test backend integration tests"
 	$(GO) test -count=1 -run "^TestIntegration" -covermode=atomic -timeout=5m $(GO_INTEGRATION_TESTS)
 
+.PHONY: test-go-integration-alertmanager
+test-go-integration-alertmanager: ## Run integration tests for the remote alertmanager (config taken from the mimir_backend block).
+	@echo "test remote alertmanager integration tests"
+	$(GO) clean -testcache
+	AM_URL=http://localhost:8080 AM_TENANT_ID=test AM_PASSWORD=test \
+	$(GO) test -count=1 -run "^TestIntegrationRemoteAlertmanager" -covermode=atomic -timeout=5m ./pkg/services/ngalert/notifier/...
+
 .PHONY: test-go-integration-postgres
 test-go-integration-postgres: devenv-postgres ## Run integration tests for postgres backend with flags.
 	@echo "test backend integration postgres tests"

pkg/services/ngalert/notifier/external_alertmanager_test.go

@@ -2,11 +2,14 @@ package notifier
 
 import (
 	"context"
+	"math/rand"
+	"os"
 	"testing"
+	"time"
 
 	"github.com/go-openapi/strfmt"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
-	amfake "github.com/grafana/grafana/pkg/services/ngalert/notifier/fake"
+	"github.com/grafana/grafana/pkg/util"
 	amv2 "github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/stretchr/testify/require"
 )
@@ -85,88 +88,103 @@ func TestNewExternalAlertmanager(t *testing.T) {
 	}
 }
 
-func TestSilences(t *testing.T) {
+func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
-	const (
+	if testing.Short() {
-		tenantID = "1"
+		t.Skip("skipping integration test")
-		password = "password"
+	}
-	)
+
-	fakeAm := amfake.NewFakeExternalAlertmanager(t, tenantID, password)
+	amURL, ok := os.LookupEnv("AM_URL")
+	if !ok {
+		t.Skip("No Alertmanager URL provided")
+	}
+	tenantID := os.Getenv("AM_TENANT_ID")
+	password := os.Getenv("AM_PASSWORD")
 
-	// Using a wrong password should cause an error.
 	cfg := externalAlertmanagerConfig{
-		URL:               fakeAm.Server.URL + "/alertmanager",
+		URL:               amURL + "/alertmanager",
 		TenantID:          tenantID,
-		BasicAuthPassword: "wrongpassword",
+		BasicAuthPassword: password,
 		DefaultConfig:     validConfig,
 	}
 	am, err := newExternalAlertmanager(cfg, 1)
 	require.NoError(t, err)
 
-	_, err = am.ListSilences(context.Background(), []string{})
-	require.NotNil(t, err)
-
-	// Using the correct password should make the request succeed.
-	cfg.BasicAuthPassword = password
-	am, err = newExternalAlertmanager(cfg, 1)
-	require.NoError(t, err)
-
 	// We should have no silences at first.
 	silences, err := am.ListSilences(context.Background(), []string{})
 	require.NoError(t, err)
 	require.Equal(t, 0, len(silences))
 
 	// Creating a silence should succeed.
-	testSilence := createSilence("test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime())
+	testSilence := genSilence("test")
-	silenceID, err := am.CreateSilence(context.Background(), &testSilence)
+	id, err := am.CreateSilence(context.Background(), &testSilence)
 	require.NoError(t, err)
-	require.NotEmpty(t, silenceID)
+	require.NotEmpty(t, id)
+	testSilence.ID = id
 
 	// We should be able to retrieve a specific silence.
-	silence, err := am.GetSilence(context.Background(), silenceID)
+	silence, err := am.GetSilence(context.Background(), testSilence.ID)
 	require.NoError(t, err)
-	require.Equal(t, *testSilence.Comment, *silence.Comment)
+	require.Equal(t, testSilence.ID, *silence.ID)
-	require.Equal(t, *testSilence.CreatedBy, *silence.CreatedBy)
-	require.Equal(t, *testSilence.StartsAt, *silence.StartsAt)
-	require.Equal(t, *testSilence.EndsAt, *silence.EndsAt)
-	require.Equal(t, testSilence.Matchers, silence.Matchers)
 
 	// Trying to retrieve a non-existing silence should fail.
-	_, err = am.GetSilence(context.Background(), "invalid")
+	_, err = am.GetSilence(context.Background(), util.GenerateShortUID())
 	require.Error(t, err)
 
 	// After creating another silence, the total amount should be 2.
-	testSilence2 := createSilence("another test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime())
+	testSilence2 := genSilence("test")
-	silenceID2, err := am.CreateSilence(context.Background(), &testSilence2)
+	id, err = am.CreateSilence(context.Background(), &testSilence2)
 	require.NoError(t, err)
-	require.NotEmpty(t, silenceID2)
+	require.NotEmpty(t, id)
+	testSilence2.ID = id
 
 	silences, err = am.ListSilences(context.Background(), []string{})
 	require.NoError(t, err)
 	require.Equal(t, 2, len(silences))
-	require.True(t, *silences[0].ID == silenceID || *silences[0].ID == silenceID2)
+	require.True(t, *silences[0].ID == testSilence.ID || *silences[0].ID == testSilence2.ID)
-	require.True(t, *silences[1].ID == silenceID || *silences[1].ID == silenceID2)
+	require.True(t, *silences[1].ID == testSilence.ID || *silences[1].ID == testSilence2.ID)
 
-	// After deleting one of those silences, the total amount should be 2.
+	// After deleting one of those silences, the total amount should be 2 but one of those should be expired.
-	err = am.DeleteSilence(context.Background(), silenceID)
+	err = am.DeleteSilence(context.Background(), testSilence.ID)
 	require.NoError(t, err)
 
 	silences, err = am.ListSilences(context.Background(), []string{})
 	require.NoError(t, err)
-	require.Equal(t, 1, len(silences))
 
-	// Trying to delete the same error should fail.
+	for _, s := range silences {
-	err = am.DeleteSilence(context.Background(), silenceID)
+		if *s.ID == testSilence.ID {
-	require.NotNil(t, err)
+			require.Equal(t, *s.Status.State, "expired")
+		} else {
+			require.Equal(t, *s.Status.State, "pending")
+		}
+	}
+
+	// When deleting the other silence, both should be expired.
+	err = am.DeleteSilence(context.Background(), testSilence2.ID)
+	require.NoError(t, err)
+
+	silences, err = am.ListSilences(context.Background(), []string{})
+	require.NoError(t, err)
+	require.Equal(t, *silences[0].Status.State, "expired")
+	require.Equal(t, *silences[1].Status.State, "expired")
 }
 
-func createSilence(comment, createdBy string, matchers amv2.Matchers, startsAt, endsAt strfmt.DateTime) apimodels.PostableSilence {
+func genSilence(createdBy string) apimodels.PostableSilence {
+	starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second))
+	ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second))
+	comment := "test comment"
+	isEqual := true
+	name := "test"
+	value := "test"
+	isRegex := false
+	matchers := amv2.Matchers{&amv2.Matcher{IsEqual: &isEqual, Name: &name, Value: &value, IsRegex: &isRegex}}
+
 	return apimodels.PostableSilence{
 		Silence: amv2.Silence{
 			Comment:   &comment,
 			CreatedBy: &createdBy,
 			Matchers:  matchers,
-			StartsAt:  &startsAt,
+			StartsAt:  &starts,
-			EndsAt:    &endsAt,
+			EndsAt:    &ends,
 		},
 	}
 }

pkg/services/ngalert/notifier/fake/external_alertmanager_fake.go

@@ -1,154 +0,0 @@
-package fake
-
-import (
-	"encoding/json"
-	"net/http"
-	"net/http/httptest"
-	"sync"
-	"testing"
-
-	"github.com/go-openapi/strfmt"
-	alertingNotify "github.com/grafana/alerting/notify"
-	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
-	"github.com/grafana/grafana/pkg/util"
-	"github.com/grafana/grafana/pkg/web"
-	"github.com/stretchr/testify/require"
-)
-
-type FakeExternalAlertmanager struct {
-	t        *testing.T
-	mtx      sync.RWMutex
-	tenantID string
-	password string
-	Server   *httptest.Server
-
-	silences alertingNotify.GettableSilences
-}
-
-func NewFakeExternalAlertmanager(t *testing.T, tenantID, password string) *FakeExternalAlertmanager {
-	t.Helper()
-	am := &FakeExternalAlertmanager{
-		t:        t,
-		tenantID: tenantID,
-		password: password,
-		mtx:      sync.RWMutex{},
-	}
-	mux := web.New()
-	mux.SetURLPrefix("/alertmanager/api/")
-	mux.UseMiddleware(am.basicAuthMiddleware)
-	mux.UseMiddleware(am.contentTypeJSONMiddleware)
-
-	// Routes
-	mux.Get("/v2/silences", http.HandlerFunc(am.getSilences))
-	mux.Get("/v2/silence/:silenceID", http.HandlerFunc(am.getSilence))
-	mux.Post("/v2/silences", http.HandlerFunc(am.postSilence))
-	mux.Delete("/v2/silence/:silenceID", http.HandlerFunc(am.deleteSilence))
-
-	am.Server = httptest.NewServer(mux)
-	return am
-}
-
-func (am *FakeExternalAlertmanager) getSilences(w http.ResponseWriter, r *http.Request) {
-	am.mtx.RLock()
-	if err := json.NewEncoder(w).Encode(am.silences); err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-	}
-	am.mtx.RUnlock()
-}
-
-func (am *FakeExternalAlertmanager) getSilence(w http.ResponseWriter, r *http.Request) {
-	silenceID, ok := web.Params(r)[":silenceID"]
-	if !ok {
-		return
-	}
-
-	am.mtx.RLock()
-	var matching *alertingNotify.GettableSilence
-	for _, silence := range am.silences {
-		if *silence.ID == silenceID {
-			matching = silence
-			break
-		}
-	}
-	am.mtx.RUnlock()
-
-	if matching == nil {
-		w.WriteHeader(http.StatusNotFound)
-		return
-	}
-
-	if err := json.NewEncoder(w).Encode(matching); err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-	}
-}
-
-func (am *FakeExternalAlertmanager) postSilence(w http.ResponseWriter, r *http.Request) {
-	var silence definitions.PostableSilence
-	require.NoError(am.t, json.NewDecoder(r.Body).Decode(&silence))
-
-	updatedAt := strfmt.NewDateTime()
-	id := util.GenerateShortUID()
-
-	am.mtx.Lock()
-	am.silences = append(am.silences, &alertingNotify.GettableSilence{
-		ID:        &id,
-		UpdatedAt: &updatedAt,
-		Silence:   silence.Silence,
-	})
-	am.mtx.Unlock()
-
-	res := map[string]string{"silenceID": id}
-	if err := json.NewEncoder(w).Encode(res); err != nil {
-		w.WriteHeader(http.StatusInternalServerError)
-	}
-}
-
-func (am *FakeExternalAlertmanager) deleteSilence(w http.ResponseWriter, r *http.Request) {
-	silenceID, ok := web.Params(r)[":silenceID"]
-	if !ok {
-		return
-	}
-
-	am.mtx.Lock()
-	defer am.mtx.Unlock()
-	var newSilences []*alertingNotify.GettableSilence
-	for _, silence := range am.silences {
-		if *silence.ID != silenceID {
-			newSilences = append(newSilences, silence)
-		}
-	}
-
-	if len(newSilences) == len(am.silences) {
-		w.WriteHeader(http.StatusNotFound)
-		return
-	}
-
-	am.silences = newSilences
-	w.WriteHeader(http.StatusOK)
-}
-
-func (am *FakeExternalAlertmanager) basicAuthMiddleware(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		username, password, ok := r.BasicAuth()
-		if !ok {
-			w.WriteHeader(http.StatusUnauthorized)
-			return
-		}
-		if username != am.tenantID || password != am.password || r.Header.Get("X-Scope-OrgID") != am.tenantID {
-			w.WriteHeader(http.StatusForbidden)
-			return
-		}
-		next.ServeHTTP(w, r)
-	})
-}
-
-func (am *FakeExternalAlertmanager) contentTypeJSONMiddleware(next http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		w.Header().Set("Content-Type", "application/json")
-		next.ServeHTTP(w, r)
-	})
-}
-
-func (am *FakeExternalAlertmanager) Close() {
-	am.Server.Close()
-}

scripts/drone/services/services.star

@@ -54,7 +54,7 @@ def integration_test_services():
             "commands": ["docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"],
         },
         {
-            "name": "mimir",
+            "name": "mimir_backend",
             "image": images["mimir"],
             "environment": {},
             "commands": ["/bin/mimir -target=backend"],

scripts/drone/steps/lib.star

@@ -964,10 +964,16 @@ def redis_integration_tests_steps():
 def remote_alertmanager_integration_tests_steps():
     cmds = [
         "go clean -testcache",
-        "go test -run IntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/...",
+        "go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...",
     ]
 
-    return integration_tests_steps("remote-alertmanager", cmds, "mimir", "8080", None)
+    environment = {
+        "AM_TENANT_ID": "test",
+        "AM_PASSWORD": "test",
+        "AM_URL": "http://mimir_backend:8080",
+    }
+
+    return integration_tests_steps("remote-alertmanager", cmds, "mimir_backend", "8080", environment = environment)
 
 def memcached_integration_tests_steps():
     cmds = [