Alerting: Add client configuration for remote Loki historian backend and test connection (#61114)

* Create loki client type and ping method

* Expose TestConnection on client

* Configure and ping Loki URL

* Close response body reader if present

* Add 30 second timeout

* Remove duplicate close

pkg/services/ngalert/ngalert.go

@@ -387,7 +387,15 @@ func configureHistorianBackend(cfg setting.UnifiedAlertingStateHistorySettings,
 		return historian.NewAnnotationBackend(ar, ds), nil
 	}
 	if cfg.Backend == "loki" {
-		return historian.NewRemoteLokiBackend(), nil
+		baseURL, err := url.Parse(cfg.LokiRemoteURL)
+		if err != nil {
+			return nil, fmt.Errorf("failed to parse remote loki URL: %w", err)
+		}
+		backend := historian.NewRemoteLokiBackend(baseURL)
+		if err := backend.TestConnection(); err != nil {
+			return nil, fmt.Errorf("failed to ping the remote loki historian: %w", err)
+		}
+		return backend, nil
 	}
 	if cfg.Backend == "sql" {
 		return historian.NewSqlBackend(), nil

pkg/services/ngalert/state/historian/loki.go

@@ -2,21 +2,35 @@ package historian
 
 import (
 	"context"
+	"net/url"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 )
 
-type RemoteLokiBackend struct {
-	log log.Logger
+type remoteLokiClient interface {
+	ping() error
 }
 
-func NewRemoteLokiBackend() *RemoteLokiBackend {
+type RemoteLokiBackend struct {
+	client remoteLokiClient
+	log    log.Logger
+}
+
+func NewRemoteLokiBackend(url *url.URL) *RemoteLokiBackend {
+	logger := log.New("ngalert.state.historian", "backend", "loki")
 	return &RemoteLokiBackend{
-		log: log.New("ngalert.state.historian"),
+		client: newLokiClient(url, logger),
+		log:    logger,
 	}
 }
 
-func (h *RemoteLokiBackend) RecordStatesAsync(ctx context.Context, _ *models.AlertRule, _ []state.StateTransition) {
+func (h *RemoteLokiBackend) TestConnection() error {
+	return h.client.ping()
+}
+
+func (h *RemoteLokiBackend) RecordStatesAsync(ctx context.Context, _ *models.AlertRule, _ []state.StateTransition) {
+	logger := h.log.FromContext(ctx)
+	logger.Debug("Remote Loki state history backend was called with states")
 }

pkg/services/ngalert/state/historian/loki_http.go

@@ -0,0 +1,54 @@
+package historian
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/grafana/grafana/pkg/infra/log"
+)
+
+const defaultClientTimeout = 30 * time.Second
+
+type httpLokiClient struct {
+	client http.Client
+	url    *url.URL
+	log    log.Logger
+}
+
+func newLokiClient(u *url.URL, logger log.Logger) *httpLokiClient {
+	return &httpLokiClient{
+		client: http.Client{
+			Timeout: defaultClientTimeout,
+		},
+		url: u,
+		log: logger.New("protocol", "http"),
+	}
+}
+
+func (c *httpLokiClient) ping() error {
+	uri := c.url.JoinPath("/loki/api/v1/status/buildinfo")
+	req, err := http.NewRequest(http.MethodGet, uri.String(), nil)
+	if err != nil {
+		return fmt.Errorf("error creating request: %w", err)
+	}
+
+	res, err := c.client.Do(req)
+	if res != nil {
+		defer func() {
+			if err := res.Body.Close(); err != nil {
+				c.log.Warn("Failed to close response body", "err", err)
+			}
+		}()
+	}
+	if err != nil {
+		return fmt.Errorf("error sending request: %w", err)
+	}
+
+	if res.StatusCode < 200 || res.StatusCode >= 300 {
+		return fmt.Errorf("request to the loki buildinfo endpoint returned a non-200 status code: %d", res.StatusCode)
+	}
+	c.log.Debug("Request to Loki buildinfo endpoint succeeded", "status", res.StatusCode)
+	return nil
+}

pkg/setting/setting_unified_alerting.go

@@ -101,8 +101,9 @@ type UnifiedAlertingReservedLabelSettings struct {
 }
 
 type UnifiedAlertingStateHistorySettings struct {
-	Enabled bool
-	Backend string
+	Enabled       bool
+	Backend       string
+	LokiRemoteURL string
 }
 
 // IsEnabled returns true if UnifiedAlertingSettings.Enabled is either nil or true.
@@ -313,8 +314,9 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 
 	stateHistory := iniFile.Section("unified_alerting.state_history")
 	uaCfgStateHistory := UnifiedAlertingStateHistorySettings{
-		Enabled: stateHistory.Key("enabled").MustBool(stateHistoryDefaultEnabled),
-		Backend: stateHistory.Key("backend").MustString("annotations"),
+		Enabled:       stateHistory.Key("enabled").MustBool(stateHistoryDefaultEnabled),
+		Backend:       stateHistory.Key("backend").MustString("annotations"),
+		LokiRemoteURL: stateHistory.Key("loki_remote_url").MustString(""),
 	}
 	uaCfg.StateHistory = uaCfgStateHistory