Alerting: Support Unified Alerting with Grafana HA (#37920)

* Alerting: Support Unified Alerting in Grafana's HA mode.

pkg/setting/setting.go

@@ -18,15 +18,14 @@ import (
 	"strings"
 	"time"
 
-	"github.com/gobwas/glob"
-
-	"github.com/prometheus/common/model"
-	"gopkg.in/ini.v1"
-
 	"github.com/grafana/grafana-aws-sdk/pkg/awsds"
 	"github.com/grafana/grafana/pkg/components/gtime"
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/util"
+
+	"github.com/gobwas/glob"
+	"github.com/prometheus/common/model"
+	"gopkg.in/ini.v1"
 )
 
 type Scheme string
@@ -420,7 +419,14 @@ type Cfg struct {
 	GeomapEnableCustomBaseLayers bool
 
 	// Unified Alerting
-	AdminConfigPollInterval time.Duration
+	AdminConfigPollInterval        time.Duration
+	AlertmanagerConfigPollInterval time.Duration
+	HAListenAddr                   string
+	HAAdvertiseAddr                string
+	HAPeers                        []string
+	HAPeerTimeout                  time.Duration
+	HAGossipInterval               time.Duration
+	HAPushPullInterval             time.Duration
 }
 
 // IsLiveConfigEnabled returns true if live should be able to save configs to SQL tables
@@ -916,8 +922,7 @@ func (cfg *Cfg) Load(args CommandLineArgs) error {
 	if err := readAlertingSettings(iniFile); err != nil {
 		return err
 	}
-
-	if err := cfg.readUnifiedAlertingSettings(iniFile); err != nil {
+	if err := cfg.ReadUnifiedAlertingSettings(iniFile); err != nil {
 		return err
 	}
 
@@ -1374,13 +1379,6 @@ func (cfg *Cfg) readRenderingSettings(iniFile *ini.File) error {
 	return nil
 }
 
-func (cfg *Cfg) readUnifiedAlertingSettings(iniFile *ini.File) error {
-	ua := iniFile.Section("unified_alerting")
-	s := ua.Key("admin_config_poll_interval_seconds").MustInt(60)
-	cfg.AdminConfigPollInterval = time.Second * time.Duration(s)
-	return nil
-}
-
 func readAlertingSettings(iniFile *ini.File) error {
 	alerting := iniFile.Section("alerting")
 	AlertingEnabled = alerting.Key("enabled").MustBool(true)

pkg/setting/setting_unified_alerting.go

@@ -0,0 +1,57 @@
+package setting
+
+import (
+	"strings"
+	"time"
+
+	"github.com/grafana/grafana/pkg/components/gtime"
+
+	"github.com/prometheus/alertmanager/cluster"
+	"gopkg.in/ini.v1"
+)
+
+const (
+	AlertmanagerDefaultClusterAddr          = "0.0.0.0:9094"
+	AlertmanagerDefaultPeerTimeout          = 15 * time.Second
+	AlertmanagerDefaultGossipInterval       = cluster.DefaultGossipInterval
+	AlertmanagerDefaultPushPullInterval     = cluster.DefaultPushPullInterval
+	SchedulerDefaultAdminConfigPollInterval = 60 * time.Second
+	AlertmanagerDefaultConfigPollInterval   = 60 * time.Second
+)
+
+func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
+	ua := iniFile.Section("unified_alerting")
+	var err error
+	cfg.AdminConfigPollInterval, err = gtime.ParseDuration(valueAsString(ua, "admin_config_poll_interval", (SchedulerDefaultAdminConfigPollInterval).String()))
+	if err != nil {
+		return err
+	}
+	cfg.AlertmanagerConfigPollInterval, err = gtime.ParseDuration(valueAsString(ua, "alertmanager_config_poll_interval", (AlertmanagerDefaultConfigPollInterval).String()))
+	if err != nil {
+		return err
+	}
+	cfg.HAPeerTimeout, err = gtime.ParseDuration(valueAsString(ua, "ha_peer_timeout", (AlertmanagerDefaultPeerTimeout).String()))
+	if err != nil {
+		return err
+	}
+	cfg.HAGossipInterval, err = gtime.ParseDuration(valueAsString(ua, "ha_gossip_interval", (AlertmanagerDefaultGossipInterval).String()))
+	if err != nil {
+		return err
+	}
+	cfg.HAPushPullInterval, err = gtime.ParseDuration(valueAsString(ua, "ha_push_pull_interval", (AlertmanagerDefaultPushPullInterval).String()))
+	if err != nil {
+		return err
+	}
+	cfg.HAListenAddr = ua.Key("ha_listen_address").MustString(AlertmanagerDefaultClusterAddr)
+	cfg.HAAdvertiseAddr = ua.Key("ha_advertise_address").MustString("")
+	peers := ua.Key("ha_peers").MustString("")
+	cfg.HAPeers = make([]string, 0)
+	if peers != "" {
+		for _, peer := range strings.Split(peers, ",") {
+			peer = strings.TrimSpace(peer)
+			cfg.HAPeers = append(cfg.HAPeers, peer)
+		}
+	}
+
+	return nil
+}

pkg/setting/setting_unified_alerting_test.go

@@ -0,0 +1,39 @@
+package setting
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestCfg_ReadUnifiedAlertingSettings(t *testing.T) {
+	cfg := NewCfg()
+	err := cfg.Load(CommandLineArgs{HomePath: "../../", Config: "../../conf/defaults.ini"})
+	require.NoError(t, err)
+
+	// It sets the correct defaults.
+	{
+		require.Equal(t, 60*time.Second, cfg.AdminConfigPollInterval)
+		require.Equal(t, 60*time.Second, cfg.AlertmanagerConfigPollInterval)
+		require.Equal(t, 15*time.Second, cfg.HAPeerTimeout)
+		require.Equal(t, "0.0.0.0:9094", cfg.HAListenAddr)
+		require.Equal(t, "", cfg.HAAdvertiseAddr)
+		require.Len(t, cfg.HAPeers, 0)
+		require.Equal(t, 200*time.Millisecond, cfg.HAGossipInterval)
+		require.Equal(t, 60*time.Second, cfg.HAPushPullInterval)
+	}
+
+	// With peers set, it correctly parses them.
+	{
+		require.Len(t, cfg.HAPeers, 0)
+		s, err := cfg.Raw.NewSection("unified_alerting")
+		require.NoError(t, err)
+		_, err = s.NewKey("ha_peers", "hostname1:9090,hostname2:9090,hostname3:9090")
+		require.NoError(t, err)
+
+		require.NoError(t, cfg.ReadUnifiedAlertingSettings(cfg.Raw))
+		require.Len(t, cfg.HAPeers, 3)
+		require.ElementsMatch(t, []string{"hostname1:9090", "hostname2:9090", "hostname3:9090"}, cfg.HAPeers)
+	}
+}