mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
Alerting: Instrument outgoing state history requests using weaveworks/common (#63600)
* Loki backend and client depend on a requester * Instrument all requests to loki using weaveworks TimedClient * Construct collector in metrics package
This commit is contained in:
Alexander Weaver
GitHub
parent
ca4cd85504
commit
e77621649d
2
go.mod
2
go.mod
@ -332,6 +332,8 @@ require (
|
||||
github.com/unknwon/bra v0.0.0-20200517080246-1e3013ecaff8 // indirect
|
||||
github.com/unknwon/com v1.0.1 // indirect
|
||||
github.com/unknwon/log v0.0.0-20150304194804-e617c87089d3 // indirect
|
||||
github.com/weaveworks/common v0.0.0-20230208133027-16871410fca4 // indirect
|
||||
github.com/weaveworks/promrus v1.2.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v0.34.0 // indirect
|
||||
go.starlark.net v0.0.0-20221020143700-22309ac47eac // indirect
|
||||
gopkg.in/fsnotify/fsnotify.v1 v1.4.7 // indirect
|
||||
|
||||
9
go.sum
9
go.sum
@ -1042,6 +1042,7 @@ github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
|
||||
github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gofrs/uuid v4.3.1+incompatible h1:0/KbAdpx3UXAx1kEOWHJeOkpbgRFGHVgv+CFIY7dBJI=
|
||||
github.com/gofrs/uuid v4.3.1+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
|
||||
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
|
||||
github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
|
||||
github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=
|
||||
github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
|
||||
@ -1055,6 +1056,7 @@ github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP
|
||||
github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
|
||||
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/gogo/status v1.0.3/go.mod h1:SavQ51ycCLnc7dGyJxp8YAmudx8xqiVrRf+6IXRsugc=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
@ -1696,6 +1698,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
|
||||
github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE=
|
||||
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.22/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
|
||||
@ -1867,6 +1870,7 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.m
|
||||
github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
|
||||
github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
|
||||
github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo=
|
||||
github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc=
|
||||
github.com/opentracing-contrib/go-grpc v0.0.0-20191001143057-db30781987df/go.mod h1:DYR5Eij8rJl8h7gblRrOZ8g0kW1umSpKqYIBTgeDtLo=
|
||||
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
|
||||
github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w=
|
||||
@ -2078,6 +2082,7 @@ github.com/segmentio/encoding v0.3.5 h1:UZEiaZ55nlXGDL92scoVuw00RmiRCazIEmvPSbSv
|
||||
github.com/segmentio/encoding v0.3.5/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM=
|
||||
github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
|
||||
github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo=
|
||||
github.com/sercand/kuberesolver v2.4.0+incompatible/go.mod h1:lWF3GL0xptCB/vCiJPl/ZshwPsX/n4Y7u0CW9E7aQIQ=
|
||||
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
|
||||
github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM=
|
||||
github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ=
|
||||
@ -2247,6 +2252,10 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF
|
||||
github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
|
||||
github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
|
||||
github.com/wadey/gocovmerge v0.0.0-20160331181800-b5bfa59ec0ad/go.mod h1:Hy8o65+MXnS6EwGElrSRjUzQDLXreJlzYLlWiHtt8hM=
|
||||
github.com/weaveworks/common v0.0.0-20230208133027-16871410fca4 h1:8eoXaryYVOWJZCnCzULYXtxiHHLrJpvoD7p283ogmo8=
|
||||
github.com/weaveworks/common v0.0.0-20230208133027-16871410fca4/go.mod h1:KoQ+3z63GUJzQ7AhU0AWQNU+LPda2EwL/cx1PlbDzVQ=
|
||||
github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M=
|
||||
github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA=
|
||||
github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
|
||||
|
||||
23
pkg/services/ngalert/metrics/historian.go
Normal file
23
pkg/services/ngalert/metrics/historian.go
Normal file
@ -0,0 +1,23 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"github.com/weaveworks/common/instrument"
|
||||
)
|
||||
|
||||
type Historian struct {
|
||||
WriteDuration *instrument.HistogramCollector
|
||||
}
|
||||
|
||||
func NewHistorianMetrics(r prometheus.Registerer) *Historian {
|
||||
return &Historian{
|
||||
WriteDuration: instrument.NewHistogramCollector(promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: Namespace,
|
||||
Subsystem: Subsystem,
|
||||
Name: "state_history_request_duration_seconds",
|
||||
Help: "Histogram of request durations to the state history store.",
|
||||
Buckets: instrument.DefBuckets,
|
||||
}, instrument.HistogramCollectorBuckets)),
|
||||
}
|
||||
}
|
||||
@ -29,6 +29,7 @@ type NGAlert struct {
|
||||
stateMetrics *State
|
||||
multiOrgAlertmanagerMetrics *MultiOrgAlertmanager
|
||||
apiMetrics *API
|
||||
historianMetrics *Historian
|
||||
}
|
||||
|
||||
// NewNGAlert manages the metrics of all the alerting components.
|
||||
@ -39,6 +40,7 @@ func NewNGAlert(r prometheus.Registerer) *NGAlert {
|
||||
stateMetrics: NewStateMetrics(r),
|
||||
multiOrgAlertmanagerMetrics: NewMultiOrgAlertmanagerMetrics(r),
|
||||
apiMetrics: NewAPIMetrics(r),
|
||||
historianMetrics: NewHistorianMetrics(r),
|
||||
}
|
||||
}
|
||||
|
||||
@ -57,3 +59,7 @@ func (ng *NGAlert) GetAPIMetrics() *API {
|
||||
func (ng *NGAlert) GetMultiOrgAlertmanagerMetrics() *MultiOrgAlertmanager {
|
||||
return ng.multiOrgAlertmanagerMetrics
|
||||
}
|
||||
|
||||
func (ng *NGAlert) GetHistorianMetrics() *Historian {
|
||||
return ng.historianMetrics
|
||||
}
|
||||
|
||||
@ -210,7 +210,7 @@ func (ng *AlertNG) init() error {
|
||||
Tracer: ng.tracer,
|
||||
}
|
||||
|
||||
history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store)
|
||||
history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -389,7 +389,7 @@ type Historian interface {
|
||||
state.Historian
|
||||
}
|
||||
|
||||
func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore) (Historian, error) {
|
||||
func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian) (Historian, error) {
|
||||
if !cfg.Enabled {
|
||||
return historian.NewNopHistorian(), nil
|
||||
}
|
||||
@ -402,7 +402,8 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid remote loki configuration: %w", err)
|
||||
}
|
||||
backend := historian.NewRemoteLokiBackend(lcfg)
|
||||
req := historian.NewRequester()
|
||||
backend := historian.NewRemoteLokiBackend(lcfg, req, met)
|
||||
|
||||
testConnCtx, cancelFunc := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancelFunc()
|
||||
|
||||
@ -8,10 +8,12 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/data"
|
||||
"github.com/weaveworks/common/http/client"
|
||||
|
||||
"github.com/grafana/grafana/pkg/components/simplejson"
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/eval"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/models"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/state"
|
||||
history_model "github.com/grafana/grafana/pkg/services/ngalert/state/historian/model"
|
||||
@ -44,14 +46,16 @@ type remoteLokiClient interface {
|
||||
type RemoteLokiBackend struct {
|
||||
client remoteLokiClient
|
||||
externalLabels map[string]string
|
||||
metrics *metrics.Historian
|
||||
log log.Logger
|
||||
}
|
||||
|
||||
func NewRemoteLokiBackend(cfg LokiConfig) *RemoteLokiBackend {
|
||||
func NewRemoteLokiBackend(cfg LokiConfig, req client.Requester, metrics *metrics.Historian) *RemoteLokiBackend {
|
||||
logger := log.New("ngalert.state.historian", "backend", "loki")
|
||||
return &RemoteLokiBackend{
|
||||
client: newLokiClient(cfg, logger),
|
||||
client: newLokiClient(cfg, req, metrics, logger),
|
||||
externalLabels: cfg.ExternalLabels,
|
||||
metrics: metrics,
|
||||
log: logger,
|
||||
}
|
||||
}
|
||||
|
||||
@ -12,11 +12,19 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/weaveworks/common/http/client"
|
||||
)
|
||||
|
||||
const defaultClientTimeout = 30 * time.Second
|
||||
|
||||
func NewRequester() client.Requester {
|
||||
return &http.Client{
|
||||
Timeout: defaultClientTimeout,
|
||||
}
|
||||
}
|
||||
|
||||
type LokiConfig struct {
|
||||
ReadPathURL *url.URL
|
||||
WritePathURL *url.URL
|
||||
@ -54,7 +62,7 @@ func NewLokiConfig(cfg setting.UnifiedAlertingStateHistorySettings) (LokiConfig,
|
||||
}
|
||||
|
||||
type httpLokiClient struct {
|
||||
client http.Client
|
||||
client client.Requester
|
||||
cfg LokiConfig
|
||||
log log.Logger
|
||||
}
|
||||
@ -81,13 +89,12 @@ type Selector struct {
|
||||
Value string
|
||||
}
|
||||
|
||||
func newLokiClient(cfg LokiConfig, logger log.Logger) *httpLokiClient {
|
||||
func newLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger) *httpLokiClient {
|
||||
tc := client.NewTimedClient(req, metrics.WriteDuration)
|
||||
return &httpLokiClient{
|
||||
client: http.Client{
|
||||
Timeout: defaultClientTimeout,
|
||||
},
|
||||
cfg: cfg,
|
||||
log: logger.New("protocol", "http"),
|
||||
client: tc,
|
||||
cfg: cfg,
|
||||
log: logger.New("protocol", "http"),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -7,7 +7,9 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
|
||||
"github.com/grafana/grafana/pkg/setting"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log"
|
||||
@ -84,7 +86,7 @@ func TestLokiHTTPClient(t *testing.T) {
|
||||
client := newLokiClient(LokiConfig{
|
||||
ReadPathURL: url,
|
||||
WritePathURL: url,
|
||||
}, log.NewNopLogger())
|
||||
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry()), log.NewNopLogger())
|
||||
|
||||
// Unauthorized request should fail against Grafana Cloud.
|
||||
err = client.ping(context.Background())
|
||||
@ -111,7 +113,7 @@ func TestLokiHTTPClient(t *testing.T) {
|
||||
WritePathURL: url,
|
||||
BasicAuthUser: "<your_username>",
|
||||
BasicAuthPassword: "<your_password>",
|
||||
}, log.NewNopLogger())
|
||||
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry()), log.NewNopLogger())
|
||||
|
||||
// When running on prem, you might need to set the tenant id,
|
||||
// so the x-scope-orgid header is set.
|
||||
|
||||
Loading…
Reference in New Issue
Block a user