Alerting: Send alerts to the remote Alertmanager (#77034)

* Alerting: Rename remote.ExternalAlertmanager to remote.Alertmanager

* Alerting: Send alerts to the remote Alertmanager

* add ticker to readiness check, add tests

* use options when creating a new sender.ExternaAlertmanager

* unexport defaultMaxQueueCapacity

* delete unused defaultConfig field

* add debug log line when sending alerts to the remote alertmanager

* move and refactor readiness check

* update tests to not include defaultConfig

pkg/services/ngalert/sender/router.go

@@ -188,10 +188,10 @@ func (d *AlertsRouter) SyncAndApplyConfigFromDatabase() error {
 	return nil
 }
 
-func buildRedactedAMs(l log.Logger, alertmanagers []externalAMcfg, ordId int64) []string {
+func buildRedactedAMs(l log.Logger, alertmanagers []ExternalAMcfg, ordId int64) []string {
 	var redactedAMs []string
 	for _, am := range alertmanagers {
-		parsedAM, err := url.Parse(am.amURL)
+		parsedAM, err := url.Parse(am.URL)
 		if err != nil {
 			l.Error("Failed to parse alertmanager string", "org", ordId, "error", err)
 			continue
@@ -208,9 +208,9 @@ func asSHA256(strings []string) string {
 	return fmt.Sprintf("%x", h.Sum(nil))
 }
 
-func (d *AlertsRouter) alertmanagersFromDatasources(orgID int64) ([]externalAMcfg, error) {
+func (d *AlertsRouter) alertmanagersFromDatasources(orgID int64) ([]ExternalAMcfg, error) {
 	var (
-		alertmanagers []externalAMcfg
+		alertmanagers []ExternalAMcfg
 	)
 	// We might have alertmanager datasources that are acting as external
 	// alertmanager, let's fetch them.
@@ -246,9 +246,9 @@ func (d *AlertsRouter) alertmanagersFromDatasources(orgID int64) ([]externalAMcf
 				"error", err)
 			continue
 		}
-		alertmanagers = append(alertmanagers, externalAMcfg{
-			amURL:   amURL,
-			headers: headers,
+		alertmanagers = append(alertmanagers, ExternalAMcfg{
+			URL:     amURL,
+			Headers: headers,
 		})
 	}
 	return alertmanagers, nil

pkg/services/ngalert/sender/router_test.go

@@ -590,10 +590,10 @@ func TestAlertManagers_buildRedactedAMs(t *testing.T) {
 
 	for _, tt := range tc {
 		t.Run(tt.name, func(t *testing.T) {
-			var cfgs []externalAMcfg
+			var cfgs []ExternalAMcfg
 			for _, url := range tt.amUrls {
-				cfgs = append(cfgs, externalAMcfg{
-					amURL: url,
+				cfgs = append(cfgs, ExternalAMcfg{
+					URL: url,
 				})
 			}
 			require.Equal(t, tt.expected, buildRedactedAMs(&fakeLogger, cfgs, tt.orgId))

pkg/services/ngalert/sender/sender.go

@@ -5,6 +5,7 @@ import (
 	"crypto/md5"
 	"errors"
 	"fmt"
+	"net/http"
 	"net/url"
 	"sort"
 	"strings"
@@ -40,35 +41,46 @@ type ExternalAlertmanager struct {
 	sdManager *discovery.Manager
 }
 
-type externalAMcfg struct {
-	amURL   string
-	headers map[string]string
+type ExternalAMcfg struct {
+	URL     string
+	Headers map[string]string
 }
 
-func (cfg *externalAMcfg) SHA256() string {
-	return asSHA256([]string{cfg.headerString(), cfg.amURL})
+type Option func(*ExternalAlertmanager)
+
+type doFunc func(context.Context, *http.Client, *http.Request) (*http.Response, error)
+
+// WithDoFunc receives a function to use when making HTTP requests from the Manager.
+func WithDoFunc(doFunc doFunc) Option {
+	return func(s *ExternalAlertmanager) {
+		s.manager.opts.Do = doFunc
+	}
+}
+
+func (cfg *ExternalAMcfg) SHA256() string {
+	return asSHA256([]string{cfg.headerString(), cfg.URL})
 }
 
 // headersString transforms all the headers in a sorted way as a
 // single string so it can be used for hashing and comparing.
-func (cfg *externalAMcfg) headerString() string {
+func (cfg *ExternalAMcfg) headerString() string {
 	var result strings.Builder
 
-	headerKeys := make([]string, 0, len(cfg.headers))
-	for key := range cfg.headers {
+	headerKeys := make([]string, 0, len(cfg.Headers))
+	for key := range cfg.Headers {
 		headerKeys = append(headerKeys, key)
 	}
 
 	sort.Strings(headerKeys)
 
 	for _, key := range headerKeys {
-		result.WriteString(fmt.Sprintf("%s:%s", key, cfg.headers[key]))
+		result.WriteString(fmt.Sprintf("%s:%s", key, cfg.Headers[key]))
 	}
 
 	return result.String()
 }
 
-func NewExternalAlertmanagerSender() *ExternalAlertmanager {
+func NewExternalAlertmanagerSender(opts ...Option) *ExternalAlertmanager {
 	l := log.New("ngalert.sender.external-alertmanager")
 	sdCtx, sdCancel := context.WithCancel(context.Background())
 	s := &ExternalAlertmanager{
@@ -85,11 +97,15 @@ func NewExternalAlertmanagerSender() *ExternalAlertmanager {
 
 	s.sdManager = discovery.NewManager(sdCtx, s.logger)
 
+	for _, opt := range opts {
+		opt(s)
+	}
+
 	return s
 }
 
 // ApplyConfig syncs a configuration with the sender.
-func (s *ExternalAlertmanager) ApplyConfig(orgId, id int64, alertmanagers []externalAMcfg) error {
+func (s *ExternalAlertmanager) ApplyConfig(orgId, id int64, alertmanagers []ExternalAMcfg) error {
 	notifierCfg, headers, err := buildNotifierConfig(alertmanagers)
 	if err != nil {
 		return err
@@ -160,11 +176,11 @@ func (s *ExternalAlertmanager) DroppedAlertmanagers() []*url.URL {
 	return s.manager.DroppedAlertmanagers()
 }
 
-func buildNotifierConfig(alertmanagers []externalAMcfg) (*config.Config, map[string]map[string]string, error) {
+func buildNotifierConfig(alertmanagers []ExternalAMcfg) (*config.Config, map[string]map[string]string, error) {
 	amConfigs := make([]*config.AlertmanagerConfig, 0, len(alertmanagers))
 	headers := map[string]map[string]string{}
 	for i, am := range alertmanagers {
-		u, err := url.Parse(am.amURL)
+		u, err := url.Parse(am.URL)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -185,10 +201,10 @@ func buildNotifierConfig(alertmanagers []externalAMcfg) (*config.Config, map[str
 			ServiceDiscoveryConfigs: sdConfig,
 		}
 
-		if am.headers != nil {
+		if am.Headers != nil {
 			// The key has the same format as the AlertmanagerConfigs.ToMap() would generate
 			// so we can use it later on when working with the alertmanager config map.
-			headers[fmt.Sprintf("config-%d", i)] = am.headers
+			headers[fmt.Sprintf("config-%d", i)] = am.Headers
 		}
 
 		// Check the URL for basic authentication information first