IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

Instrument/ruler api (#33290)

Browse Source 
* ruler api histogram instrumentation

* register ruler metrics
This commit is contained in:
Owen Diehl 2021-04-27 08:25:32 -04:00 committed by GitHub
parent be1affe0a4
commit 86c8eed386
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 56 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
go.sum
View File

@ -1028,6 +1028,7 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E
github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5PtRc=
github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=

10
pkg/services/ngalert/api/api.go
View File

@ -4,6 +4,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/services/ngalert/state"
"github.com/prometheus/client_golang/prometheus"
"github.com/go-macaron/binding"
@ -59,6 +60,14 @@ func (api *API) RegisterAPIEndpoints() {
proxy := &AlertingProxy{
DataProxy: api.DataProxy,
}
var reg prometheus.Registerer
// hack, this just assumes that if this histogram is enabled, we should enable others
// TODO(owen-d): expose this as a config option (alerting-instrumentation or similar)
if api.Cfg.IsHTTPRequestHistogramEnabled() {
reg = prometheus.DefaultRegisterer
}
// Register endpoints for proxing to Alertmanager-compatible backends.
api.RegisterAlertmanagerApiEndpoints(NewForkedAM(
api.DatasourceCache,
@ -76,6 +85,7 @@ func (api *API) RegisterAPIEndpoints() {
api.DatasourceCache,
NewLotexRuler(proxy, logger),
RulerSrv{store: api.RuleStore, log: logger},
reg,
))
api.RegisterTestingApiEndpoints(TestingApiSrv{
AlertingProxy: proxy,

59
pkg/services/ngalert/api/fork_ruler.go
View File

@ -2,26 +2,57 @@ package api
import (
"fmt"
"time"
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/datasources"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/prometheus/client_golang/prometheus"
)
const (
GrafanaBackend = "grafana"
ProxyBackend = "proxy" // metric cardinality is too high to enumerate all non-grafana backends
)
// ForkedRuler will validate and proxy requests to the correct backend type depending on the datasource.
type ForkedRuler struct {
LotexRuler, GrafanaRuler RulerApiService
DatasourceCache datasources.CacheService
// metrics
duration *prometheus.HistogramVec
}
// NewForkedRuler implements a set of routes that proxy to various Cortex Ruler-compatible backends.
func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana RulerApiService) *ForkedRuler {
return &ForkedRuler{
func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana RulerApiService, reg prometheus.Registerer) *ForkedRuler {
r := &ForkedRuler{
LotexRuler: lotex,
GrafanaRuler: grafana,
DatasourceCache: datasourceCache,
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "grafana",
Name: "alerting_ruler_api_duration_seconds",
Help: "Histogram of latencies affecting the Unified Alerting Ruler API",
Buckets: prometheus.DefBuckets,
}, []string{"backend", "status"}),
}
if reg != nil {
reg.MustRegister(
r.duration,
)
}
return r
}
func (r *ForkedRuler) instrument(backend string, fn func() response.Response) response.Response {
start := time.Now()
resp := fn()
r.duration.
WithLabelValues(backend, fmt.Sprint(resp.Status())).
Observe(time.Since(start).Seconds())
return resp
}
func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) response.Response {
@ -31,9 +62,9 @@ func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) re
}
switch t {
case apimodels.GrafanaBackend:
return r.GrafanaRuler.RouteDeleteNamespaceRulesConfig(ctx)
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteDeleteNamespaceRulesConfig(ctx) })
case apimodels.LoTexRulerBackend:
return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx)
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx) })
default:
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
}
@ -46,9 +77,9 @@ func (r *ForkedRuler) RouteDeleteRuleGroupConfig(ctx *models.ReqContext) respons
}
switch t {
case apimodels.GrafanaBackend:
return r.GrafanaRuler.RouteDeleteRuleGroupConfig(ctx)
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteDeleteRuleGroupConfig(ctx) })
case apimodels.LoTexRulerBackend:
return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx)
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx) })
default:
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
}
@ -61,9 +92,9 @@ func (r *ForkedRuler) RouteGetNamespaceRulesConfig(ctx *models.ReqContext) respo
}
switch t {
case apimodels.GrafanaBackend:
return r.GrafanaRuler.RouteGetNamespaceRulesConfig(ctx)
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetNamespaceRulesConfig(ctx) })
case apimodels.LoTexRulerBackend:
return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx)
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx) })
default:
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
}
@ -76,9 +107,9 @@ func (r *ForkedRuler) RouteGetRulegGroupConfig(ctx *models.ReqContext) response.
}
switch t {
case apimodels.GrafanaBackend:
return r.GrafanaRuler.RouteGetRulegGroupConfig(ctx)
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetRulegGroupConfig(ctx) })
case apimodels.LoTexRulerBackend:
return r.LotexRuler.RouteGetRulegGroupConfig(ctx)
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetRulegGroupConfig(ctx) })
default:
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
}
@ -91,9 +122,9 @@ func (r *ForkedRuler) RouteGetRulesConfig(ctx *models.ReqContext) response.Respo
}
switch t {
case apimodels.GrafanaBackend:
return r.GrafanaRuler.RouteGetRulesConfig(ctx)
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetRulesConfig(ctx) })
case apimodels.LoTexRulerBackend:
return r.LotexRuler.RouteGetRulesConfig(ctx)
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetRulesConfig(ctx) })
default:
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
}
@ -120,9 +151,9 @@ func (r *ForkedRuler) RoutePostNameRulesConfig(ctx *models.ReqContext, conf apim
switch backendType {
case apimodels.GrafanaBackend:
return r.GrafanaRuler.RoutePostNameRulesConfig(ctx, conf)
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RoutePostNameRulesConfig(ctx, conf) })
case apimodels.LoTexRulerBackend:
return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf)
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf) })
default:
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", backendType), nil)
}