Alerting: Add options to configure TLS for HA using Redis (#87567)

* Add Alerting HA Redis Client TLS configs

* Add test to ping miniredis with mTLS

* Update .ini files and docs

* Add tests for unified alerting ha redis TLS settings

* Fix malformed go.sum

* Add modowner

* Fix lint error

* Update docs and use dstls config

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -169,13 +169,15 @@ func (moa *MultiOrgAlertmanager) setupClustering(cfg *setting.Cfg) error {
 	// Redis setup.
 	if cfg.UnifiedAlerting.HARedisAddr != "" {
 		redisPeer, err := newRedisPeer(redisConfig{
-			addr:     cfg.UnifiedAlerting.HARedisAddr,
-			name:     cfg.UnifiedAlerting.HARedisPeerName,
-			prefix:   cfg.UnifiedAlerting.HARedisPrefix,
-			password: cfg.UnifiedAlerting.HARedisPassword,
-			username: cfg.UnifiedAlerting.HARedisUsername,
-			db:       cfg.UnifiedAlerting.HARedisDB,
-			maxConns: cfg.UnifiedAlerting.HARedisMaxConns,
+			addr:       cfg.UnifiedAlerting.HARedisAddr,
+			name:       cfg.UnifiedAlerting.HARedisPeerName,
+			prefix:     cfg.UnifiedAlerting.HARedisPrefix,
+			password:   cfg.UnifiedAlerting.HARedisPassword,
+			username:   cfg.UnifiedAlerting.HARedisUsername,
+			db:         cfg.UnifiedAlerting.HARedisDB,
+			maxConns:   cfg.UnifiedAlerting.HARedisMaxConns,
+			tlsEnabled: cfg.UnifiedAlerting.HARedisTLSEnabled,
+			tls:        cfg.UnifiedAlerting.HARedisTLSConfig,
 		}, clusterLogger, moa.metrics.Registerer, cfg.UnifiedAlerting.HAPushPullInterval)
 		if err != nil {
 			return fmt.Errorf("unable to initialize redis: %w", err)

pkg/services/ngalert/notifier/redis_peer.go

@@ -12,6 +12,7 @@ import (
 	"github.com/google/uuid"
 	alertingCluster "github.com/grafana/alerting/cluster"
 	alertingClusterPB "github.com/grafana/alerting/cluster/clusterpb"
+	dstls "github.com/grafana/dskit/crypto/tls"
 	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/redis/go-redis/v9"
@@ -27,6 +28,9 @@ type redisConfig struct {
 	name     string
 	prefix   string
 	maxConns int
+
+	tlsEnabled bool
+	tls        dstls.ClientConfig
 }
 
 const (
@@ -90,13 +94,26 @@ func newRedisPeer(cfg redisConfig, logger log.Logger, reg prometheus.Registerer,
 	if cfg.maxConns >= 0 {
 		poolSize = cfg.maxConns
 	}
-	rdb := redis.NewClient(&redis.Options{
+
+	opts := &redis.Options{
 		Addr:     cfg.addr,
 		Username: cfg.username,
 		Password: cfg.password,
 		DB:       cfg.db,
 		PoolSize: poolSize,
-	})
+	}
+
+	if cfg.tlsEnabled {
+		tlsClientConfig, err := cfg.tls.GetTLSConfig()
+		if err != nil {
+			logger.Error("Failed to get TLS config", "err", err)
+			return nil, err
+		} else {
+			opts.TLSConfig = tlsClientConfig
+		}
+	}
+
+	rdb := redis.NewClient(opts)
 	cmd := rdb.Ping(context.Background())
 	if cmd.Err() != nil {
 		logger.Error("Failed to ping redis - redis-based alertmanager clustering may not be available", "err", cmd.Err())

pkg/services/ngalert/notifier/redis_peer_test.go

@@ -0,0 +1,89 @@
+package notifier
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/alicebob/miniredis/v2"
+	dstls "github.com/grafana/dskit/crypto/tls"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/madflojo/testcerts"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/stretchr/testify/require"
+)
+
+func TestNewRedisPeerWithTLS(t *testing.T) {
+	// Write client and server certificates/keys to tempDir, both issues by the same CA
+	certPaths := createX509TestDir(t)
+
+	// Set up tls.Config and start miniredis with server-side TLS
+	x509Cert, err := tls.LoadX509KeyPair(certPaths.serverCert, certPaths.serverKey)
+	require.NoError(t, err)
+	clientCAPool := x509.NewCertPool()
+	clientCAFile, err := os.ReadFile(certPaths.ca)
+	require.NoError(t, err)
+	clientCAPool.AppendCertsFromPEM(clientCAFile)
+
+	mr, err := miniredis.RunTLS(&tls.Config{
+		Certificates: []tls.Certificate{x509Cert},
+		ClientCAs:    clientCAPool,
+	})
+	require.NoError(t, err)
+	defer mr.Close()
+
+	// Create redis peer with client-side TLS
+	redisPeer, err := newRedisPeer(redisConfig{
+		addr:       mr.Addr(),
+		tlsEnabled: true,
+		tls: dstls.ClientConfig{
+			CertPath:   certPaths.clientCert,
+			KeyPath:    certPaths.clientKey,
+			CAPath:     certPaths.ca,
+			ServerName: "localhost",
+		}}, log.NewNopLogger(), prometheus.DefaultRegisterer, time.Second*60)
+	require.NoError(t, err)
+
+	ping := redisPeer.redis.Ping(context.Background())
+	require.NoError(t, ping.Err())
+}
+
+type certPaths struct {
+	clientCert string
+	clientKey  string
+	serverCert string
+	serverKey  string
+	ca         string
+}
+
+func createX509TestDir(t *testing.T) certPaths {
+	t.Helper()
+
+	tmpDir := t.TempDir()
+
+	ca := testcerts.NewCA()
+	caCertFile, _, err := ca.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	serverKp, err := ca.NewKeyPair("localhost")
+	require.NoError(t, err)
+
+	serverCertFile, serverKeyFile, err := serverKp.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	clientKp, err := ca.NewKeyPair()
+	require.NoError(t, err)
+	clientCertFile, clientKeyFile, err := clientKp.ToTempFile(tmpDir)
+	require.NoError(t, err)
+
+	return certPaths{
+		clientCert: clientCertFile.Name(),
+		clientKey:  clientKeyFile.Name(),
+		serverCert: serverCertFile.Name(),
+		serverKey:  serverKeyFile.Name(),
+		ca:         caCertFile.Name(),
+	}
+}