Alerting: Instrument outgoing state history requests using weaveworks/common (#63600)

* Loki backend and client depend on a requester

* Instrument all requests to loki using weaveworks TimedClient

* Construct collector in metrics package

pkg/services/ngalert/metrics/historian.go

@@ -0,0 +1,23 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promauto"
+	"github.com/weaveworks/common/instrument"
+)
+
+type Historian struct {
+	WriteDuration *instrument.HistogramCollector
+}
+
+func NewHistorianMetrics(r prometheus.Registerer) *Historian {
+	return &Historian{
+		WriteDuration: instrument.NewHistogramCollector(promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: Namespace,
+			Subsystem: Subsystem,
+			Name:      "state_history_request_duration_seconds",
+			Help:      "Histogram of request durations to the state history store.",
+			Buckets:   instrument.DefBuckets,
+		}, instrument.HistogramCollectorBuckets)),
+	}
+}

pkg/services/ngalert/metrics/ngalert.go

@@ -29,6 +29,7 @@ type NGAlert struct {
 	stateMetrics                *State
 	multiOrgAlertmanagerMetrics *MultiOrgAlertmanager
 	apiMetrics                  *API
+	historianMetrics            *Historian
 }
 
 // NewNGAlert manages the metrics of all the alerting components.
@@ -39,6 +40,7 @@ func NewNGAlert(r prometheus.Registerer) *NGAlert {
 		stateMetrics:                NewStateMetrics(r),
 		multiOrgAlertmanagerMetrics: NewMultiOrgAlertmanagerMetrics(r),
 		apiMetrics:                  NewAPIMetrics(r),
+		historianMetrics:            NewHistorianMetrics(r),
 	}
 }
 
@@ -57,3 +59,7 @@ func (ng *NGAlert) GetAPIMetrics() *API {
 func (ng *NGAlert) GetMultiOrgAlertmanagerMetrics() *MultiOrgAlertmanager {
 	return ng.multiOrgAlertmanagerMetrics
 }
+
+func (ng *NGAlert) GetHistorianMetrics() *Historian {
+	return ng.historianMetrics
+}