Alerting: Add metrics to the remote Alertmanager struct (#79835)

* Alerting: Add metrics to the remote Alertmanager struct

* rephrase http_requests_failed description

* make linter happy

* remove unnecessary metrics

* extract timed client to separate package

* use histogram collector from dskit

* remove weaveworks dependency

* capture metrics for all requests to the remote Alertmanager (both clients)

* use the timed client in the MimirAuthRoundTripper

* HTTPRequestsDuration -> HTTPRequestDuration, clean up mimir client factory function

* refactor

* less git diff

* gauge for last readiness check in seconds

* initialize LastReadinesCheck to 0, tweak metric names and descriptions

* add counters for sync attempts/errors

* last config sync and last state sync timestamps (gauges)

* change latency metric name

* metric for remote Alertmanager mode

* code review comments

* move label constants to metrics package

pkg/services/ngalert/client/client.go

@@ -0,0 +1,72 @@
+package client
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"strconv"
+
+	"github.com/grafana/dskit/instrument"
+)
+
+// Requester executes an HTTP request.
+type Requester interface {
+	Do(req *http.Request) (*http.Response, error)
+}
+
+// TimedClient instruments a request. It implements Requester.
+type TimedClient struct {
+	client    Requester
+	collector instrument.Collector
+}
+
+type contextKey int
+
+// OperationNameContextKey specifies the operation name location within the context
+// for instrumentation.
+const OperationNameContextKey contextKey = 0
+
+// NewTimedClient creates a Requester that instruments requests on `client`.
+func NewTimedClient(client Requester, collector instrument.Collector) *TimedClient {
+	return &TimedClient{
+		client:    client,
+		collector: collector,
+	}
+}
+
+// Do executes the request.
+func (c TimedClient) Do(r *http.Request) (*http.Response, error) {
+	return TimeRequest(r.Context(), c.operationName(r), c.collector, c.client, r)
+}
+
+// RoundTrip implements the RoundTripper interface.
+func (c TimedClient) RoundTrip(r *http.Request) (*http.Response, error) {
+	return c.Do(r)
+}
+
+func (c TimedClient) operationName(r *http.Request) string {
+	operation, _ := r.Context().Value(OperationNameContextKey).(string)
+	if operation == "" {
+		operation = r.URL.Path
+	}
+	return operation
+}
+
+// TimeRequest performs an HTTP client request and records the duration in a histogram.
+func TimeRequest(ctx context.Context, operation string, coll instrument.Collector, client Requester, request *http.Request) (*http.Response, error) {
+	var response *http.Response
+	doRequest := func(_ context.Context) error {
+		var err error
+		response, err = client.Do(request) // nolint:bodyclose
+		return err
+	}
+	toStatusCode := func(err error) string {
+		if err == nil {
+			return strconv.Itoa(response.StatusCode)
+		}
+		return "error"
+	}
+	err := instrument.CollectedRequest(ctx, fmt.Sprintf("%s %s", request.Method, operation),
+		coll, toStatusCode, doRequest)
+	return response, err
+}

pkg/services/ngalert/client/client_test.go

@@ -0,0 +1,29 @@
+package client
+
+import (
+	"context"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestTimedClient_operationName(t *testing.T) {
+	r, err := http.NewRequest("GET", "https://weave.test", nil)
+	assert.NoError(t, err)
+
+	r = r.WithContext(context.WithValue(context.Background(), OperationNameContextKey, "opp"))
+	c := NewTimedClient(http.DefaultClient, nil)
+
+	assert.Equal(t, "opp", c.operationName(r))
+}
+
+func TestTimedClient_operationName_Default(t *testing.T) {
+	r, err := http.NewRequest("GET", "https://weave.test/you/know/me", nil)
+	assert.NoError(t, err)
+
+	r = r.WithContext(context.Background())
+	c := NewTimedClient(http.DefaultClient, nil)
+
+	assert.Equal(t, "/you/know/me", c.operationName(r))
+}