mirror of
https://github.com/grafana/grafana.git
synced 2025-02-25 18:55:37 -06:00
[Alerting] Automatic request instrumentation (#33444)
* alerting: automatic request instrumentation * always expose alerting prom metrics * globally register alerting metrics
This commit is contained in:
Owen Diehl
GitHub
parent
ef9f99645d
commit
ec37b4cb87
2
go.mod
2
go.mod
@ -95,7 +95,7 @@ require (
|
||||
golang.org/x/oauth2 v0.0.0-20210413134643-5e61552d6c78
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
|
||||
golang.org/x/tools v0.1.0 // indirect
|
||||
golang.org/x/tools v0.1.0
|
||||
gonum.org/v1/gonum v0.9.1
|
||||
google.golang.org/api v0.45.0
|
||||
google.golang.org/grpc v1.37.0
|
||||
|
||||
6
go.sum
6
go.sum
@ -51,8 +51,6 @@ code.cloudfoundry.org/bytefmt v0.0.0-20200131002437-cf55d5288a48/go.mod h1:wN/zk
|
||||
collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE=
|
||||
contrib.go.opencensus.io/exporter/ocagent v0.6.0/go.mod h1:zmKjrJcdo0aYcVS7bmEeSEBLPA9YJp5bjrofdU3pIXs=
|
||||
contrib.go.opencensus.io/exporter/prometheus v0.2.0/go.mod h1:TYmVAyE8Tn1lyPcltF5IYYfWp2KHu7lQGIZnj8iZMys=
|
||||
cuelang.org/go v0.3.0-beta.6 h1:od1S/Hbl2S45TLSONl95X3O4TXN1za6CUSD13bTxCVk=
|
||||
cuelang.org/go v0.3.0-beta.6/go.mod h1:Ikvs157igkGV5gFUdYSFa+lWp/CDteVhubPTXyvPRtA=
|
||||
cuelang.org/go v0.3.2 h1:/Am5yFDwqnaEi+g942OPM1M4/qtfVSm49wtkQbeh5Z4=
|
||||
cuelang.org/go v0.3.2/go.mod h1:jvMO35Q4D2D3m2ujAmKESICaYkjMbu5+D+2zIGuWTpQ=
|
||||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
|
||||
@ -1031,7 +1029,6 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E
|
||||
github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5PtRc=
|
||||
github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
|
||||
github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
|
||||
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
|
||||
github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
|
||||
github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
|
||||
github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
|
||||
@ -1451,8 +1448,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L
|
||||
github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rogpeppe/go-internal v1.6.2-0.20200830194709-1115b6af0369 h1:wdCVGtPadWC/ZuuLC7Hv58VQ5UF7V98ewE71n5mJfrM=
|
||||
github.com/rogpeppe/go-internal v1.6.2-0.20200830194709-1115b6af0369/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
|
||||
github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
|
||||
github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE=
|
||||
github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
|
||||
github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
|
||||
|
||||
@ -24,6 +24,11 @@ import (
|
||||
// timeNow makes it possible to test usage of time
|
||||
var timeNow = time.Now
|
||||
|
||||
// metrics are a globally registered metric suite for alerting.
|
||||
// TODO: refactor testware to allow these to be created without
|
||||
// panicking on duplicate registration, thus enabling non-global vars.
|
||||
var metrics = NewMetrics(prometheus.DefaultRegisterer)
|
||||
|
||||
type Alertmanager interface {
|
||||
// Configuration
|
||||
SaveAndApplyConfig(config *apimodels.PostableUserConfig) error
|
||||
@ -61,39 +66,31 @@ func (api *API) RegisterAPIEndpoints() {
|
||||
DataProxy: api.DataProxy,
|
||||
}
|
||||
|
||||
var reg prometheus.Registerer
|
||||
// hack, this just assumes that if this histogram is enabled, we should enable others
|
||||
// TODO(owen-d): expose this as a config option (alerting-instrumentation or similar)
|
||||
if api.Cfg.IsHTTPRequestHistogramEnabled() {
|
||||
reg = prometheus.DefaultRegisterer
|
||||
}
|
||||
|
||||
// Register endpoints for proxing to Alertmanager-compatible backends.
|
||||
api.RegisterAlertmanagerApiEndpoints(NewForkedAM(
|
||||
api.DatasourceCache,
|
||||
NewLotexAM(proxy, logger),
|
||||
AlertmanagerSrv{store: api.AlertingStore, am: api.Alertmanager, log: logger},
|
||||
))
|
||||
), metrics)
|
||||
// Register endpoints for proxing to Prometheus-compatible backends.
|
||||
api.RegisterPrometheusApiEndpoints(NewForkedProm(
|
||||
api.DatasourceCache,
|
||||
NewLotexProm(proxy, logger),
|
||||
PrometheusSrv{log: logger, manager: api.StateManager, store: api.RuleStore},
|
||||
))
|
||||
), metrics)
|
||||
// Register endpoints for proxing to Cortex Ruler-compatible backends.
|
||||
api.RegisterRulerApiEndpoints(NewForkedRuler(
|
||||
api.DatasourceCache,
|
||||
NewLotexRuler(proxy, logger),
|
||||
RulerSrv{DatasourceCache: api.DatasourceCache, store: api.RuleStore, log: logger},
|
||||
reg,
|
||||
))
|
||||
), metrics)
|
||||
api.RegisterTestingApiEndpoints(TestingApiSrv{
|
||||
AlertingProxy: proxy,
|
||||
Cfg: api.Cfg,
|
||||
DataService: api.DataService,
|
||||
DatasourceCache: api.DatasourceCache,
|
||||
log: logger,
|
||||
})
|
||||
}, metrics)
|
||||
|
||||
// Legacy routes; they will be removed in v8
|
||||
api.RouteRegister.Group("/api/alert-definitions", func(alertDefinitions routing.RouteRegister) {
|
||||
|
||||
@ -2,57 +2,26 @@ package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
"github.com/grafana/grafana/pkg/services/datasources"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
const (
|
||||
GrafanaBackend = "grafana"
|
||||
ProxyBackend = "proxy" // metric cardinality is too high to enumerate all non-grafana backends
|
||||
)
|
||||
|
||||
// ForkedRuler will validate and proxy requests to the correct backend type depending on the datasource.
|
||||
type ForkedRuler struct {
|
||||
LotexRuler, GrafanaRuler RulerApiService
|
||||
DatasourceCache datasources.CacheService
|
||||
|
||||
// metrics
|
||||
duration *prometheus.HistogramVec
|
||||
}
|
||||
|
||||
// NewForkedRuler implements a set of routes that proxy to various Cortex Ruler-compatible backends.
|
||||
func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana RulerApiService, reg prometheus.Registerer) *ForkedRuler {
|
||||
r := &ForkedRuler{
|
||||
func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana RulerApiService) *ForkedRuler {
|
||||
return &ForkedRuler{
|
||||
LotexRuler: lotex,
|
||||
GrafanaRuler: grafana,
|
||||
DatasourceCache: datasourceCache,
|
||||
duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
|
||||
Namespace: "grafana",
|
||||
Name: "alerting_ruler_api_duration_seconds",
|
||||
Help: "Histogram of latencies affecting the Unified Alerting Ruler API",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}, []string{"backend", "status"}),
|
||||
}
|
||||
if reg != nil {
|
||||
reg.MustRegister(
|
||||
r.duration,
|
||||
)
|
||||
}
|
||||
return r
|
||||
}
|
||||
|
||||
func (r *ForkedRuler) instrument(backend string, fn func() response.Response) response.Response {
|
||||
start := time.Now()
|
||||
resp := fn()
|
||||
r.duration.
|
||||
WithLabelValues(backend, fmt.Sprint(resp.Status())).
|
||||
Observe(time.Since(start).Seconds())
|
||||
return resp
|
||||
}
|
||||
|
||||
func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) response.Response {
|
||||
@ -62,9 +31,9 @@ func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) re
|
||||
}
|
||||
switch t {
|
||||
case apimodels.GrafanaBackend:
|
||||
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteDeleteNamespaceRulesConfig(ctx) })
|
||||
return r.GrafanaRuler.RouteDeleteNamespaceRulesConfig(ctx)
|
||||
case apimodels.LoTexRulerBackend:
|
||||
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx) })
|
||||
return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx)
|
||||
default:
|
||||
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
|
||||
}
|
||||
@ -77,9 +46,9 @@ func (r *ForkedRuler) RouteDeleteRuleGroupConfig(ctx *models.ReqContext) respons
|
||||
}
|
||||
switch t {
|
||||
case apimodels.GrafanaBackend:
|
||||
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteDeleteRuleGroupConfig(ctx) })
|
||||
return r.GrafanaRuler.RouteDeleteRuleGroupConfig(ctx)
|
||||
case apimodels.LoTexRulerBackend:
|
||||
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx) })
|
||||
return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx)
|
||||
default:
|
||||
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
|
||||
}
|
||||
@ -92,9 +61,9 @@ func (r *ForkedRuler) RouteGetNamespaceRulesConfig(ctx *models.ReqContext) respo
|
||||
}
|
||||
switch t {
|
||||
case apimodels.GrafanaBackend:
|
||||
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetNamespaceRulesConfig(ctx) })
|
||||
return r.GrafanaRuler.RouteGetNamespaceRulesConfig(ctx)
|
||||
case apimodels.LoTexRulerBackend:
|
||||
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx) })
|
||||
return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx)
|
||||
default:
|
||||
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
|
||||
}
|
||||
@ -107,9 +76,9 @@ func (r *ForkedRuler) RouteGetRulegGroupConfig(ctx *models.ReqContext) response.
|
||||
}
|
||||
switch t {
|
||||
case apimodels.GrafanaBackend:
|
||||
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetRulegGroupConfig(ctx) })
|
||||
return r.GrafanaRuler.RouteGetRulegGroupConfig(ctx)
|
||||
case apimodels.LoTexRulerBackend:
|
||||
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetRulegGroupConfig(ctx) })
|
||||
return r.LotexRuler.RouteGetRulegGroupConfig(ctx)
|
||||
default:
|
||||
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
|
||||
}
|
||||
@ -122,9 +91,9 @@ func (r *ForkedRuler) RouteGetRulesConfig(ctx *models.ReqContext) response.Respo
|
||||
}
|
||||
switch t {
|
||||
case apimodels.GrafanaBackend:
|
||||
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetRulesConfig(ctx) })
|
||||
return r.GrafanaRuler.RouteGetRulesConfig(ctx)
|
||||
case apimodels.LoTexRulerBackend:
|
||||
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetRulesConfig(ctx) })
|
||||
return r.LotexRuler.RouteGetRulesConfig(ctx)
|
||||
default:
|
||||
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
|
||||
}
|
||||
@ -151,9 +120,9 @@ func (r *ForkedRuler) RoutePostNameRulesConfig(ctx *models.ReqContext, conf apim
|
||||
|
||||
switch backendType {
|
||||
case apimodels.GrafanaBackend:
|
||||
return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RoutePostNameRulesConfig(ctx, conf) })
|
||||
return r.GrafanaRuler.RoutePostNameRulesConfig(ctx, conf)
|
||||
case apimodels.LoTexRulerBackend:
|
||||
return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf) })
|
||||
return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf)
|
||||
default:
|
||||
return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", backendType), nil)
|
||||
}
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-macaron/binding"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
@ -30,17 +32,100 @@ type AlertmanagerApiService interface {
|
||||
RoutePostAlertingConfig(*models.ReqContext, apimodels.PostableUserConfig) response.Response
|
||||
}
|
||||
|
||||
func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService) {
|
||||
func (api *API) RegisterAlertmanagerApiEndpoints(srv AlertmanagerApiService, metrics *Metrics) {
|
||||
api.RouteRegister.Group("", func(group routing.RouteRegister) {
|
||||
group.Post(toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silences"), binding.Bind(apimodels.PostableSilence{}), routing.Wrap(srv.RouteCreateSilence))
|
||||
group.Delete(toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"), routing.Wrap(srv.RouteDeleteAlertingConfig))
|
||||
group.Delete(toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"), routing.Wrap(srv.RouteDeleteSilence))
|
||||
group.Get(toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts/groups"), routing.Wrap(srv.RouteGetAMAlertGroups))
|
||||
group.Get(toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts"), routing.Wrap(srv.RouteGetAMAlerts))
|
||||
group.Get(toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"), routing.Wrap(srv.RouteGetAlertingConfig))
|
||||
group.Get(toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"), routing.Wrap(srv.RouteGetSilence))
|
||||
group.Get(toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silences"), routing.Wrap(srv.RouteGetSilences))
|
||||
group.Post(toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts"), binding.Bind(apimodels.PostableAlerts{}), routing.Wrap(srv.RoutePostAMAlerts))
|
||||
group.Post(toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"), binding.Bind(apimodels.PostableUserConfig{}), routing.Wrap(srv.RoutePostAlertingConfig))
|
||||
group.Post(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silences"),
|
||||
binding.Bind(apimodels.PostableSilence{}),
|
||||
Instrument(
|
||||
http.MethodPost,
|
||||
"/api/alertmanager/{Recipient}/api/v2/silences",
|
||||
srv.RouteCreateSilence,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Delete(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"),
|
||||
Instrument(
|
||||
http.MethodDelete,
|
||||
"/api/alertmanager/{Recipient}/config/api/v1/alerts",
|
||||
srv.RouteDeleteAlertingConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Delete(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"),
|
||||
Instrument(
|
||||
http.MethodDelete,
|
||||
"/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}",
|
||||
srv.RouteDeleteSilence,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts/groups"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/alertmanager/{Recipient}/api/v2/alerts/groups",
|
||||
srv.RouteGetAMAlertGroups,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/alertmanager/{Recipient}/api/v2/alerts",
|
||||
srv.RouteGetAMAlerts,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/alertmanager/{Recipient}/config/api/v1/alerts",
|
||||
srv.RouteGetAlertingConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/alertmanager/{Recipient}/api/v2/silence/{SilenceId}",
|
||||
srv.RouteGetSilence,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/silences"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/alertmanager/{Recipient}/api/v2/silences",
|
||||
srv.RouteGetSilences,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Post(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/api/v2/alerts"),
|
||||
binding.Bind(apimodels.PostableAlerts{}),
|
||||
Instrument(
|
||||
http.MethodPost,
|
||||
"/api/alertmanager/{Recipient}/api/v2/alerts",
|
||||
srv.RoutePostAMAlerts,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Post(
|
||||
toMacaronPath("/api/alertmanager/{Recipient}/config/api/v1/alerts"),
|
||||
binding.Bind(apimodels.PostableUserConfig{}),
|
||||
Instrument(
|
||||
http.MethodPost,
|
||||
"/api/alertmanager/{Recipient}/config/api/v1/alerts",
|
||||
srv.RoutePostAlertingConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
}, middleware.ReqSignedIn)
|
||||
}
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
"github.com/grafana/grafana/pkg/api/routing"
|
||||
"github.com/grafana/grafana/pkg/middleware"
|
||||
@ -19,9 +21,25 @@ type PrometheusApiService interface {
|
||||
RouteGetRuleStatuses(*models.ReqContext) response.Response
|
||||
}
|
||||
|
||||
func (api *API) RegisterPrometheusApiEndpoints(srv PrometheusApiService) {
|
||||
func (api *API) RegisterPrometheusApiEndpoints(srv PrometheusApiService, metrics *Metrics) {
|
||||
api.RouteRegister.Group("", func(group routing.RouteRegister) {
|
||||
group.Get(toMacaronPath("/api/prometheus/{Recipient}/api/v1/alerts"), routing.Wrap(srv.RouteGetAlertStatuses))
|
||||
group.Get(toMacaronPath("/api/prometheus/{Recipient}/api/v1/rules"), routing.Wrap(srv.RouteGetRuleStatuses))
|
||||
group.Get(
|
||||
toMacaronPath("/api/prometheus/{Recipient}/api/v1/alerts"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/prometheus/{Recipient}/api/v1/alerts",
|
||||
srv.RouteGetAlertStatuses,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/prometheus/{Recipient}/api/v1/rules"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/prometheus/{Recipient}/api/v1/rules",
|
||||
srv.RouteGetRuleStatuses,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
}, middleware.ReqSignedIn)
|
||||
}
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-macaron/binding"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
@ -26,13 +28,62 @@ type RulerApiService interface {
|
||||
RoutePostNameRulesConfig(*models.ReqContext, apimodels.PostableRuleGroupConfig) response.Response
|
||||
}
|
||||
|
||||
func (api *API) RegisterRulerApiEndpoints(srv RulerApiService) {
|
||||
func (api *API) RegisterRulerApiEndpoints(srv RulerApiService, metrics *Metrics) {
|
||||
api.RouteRegister.Group("", func(group routing.RouteRegister) {
|
||||
group.Delete(toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"), routing.Wrap(srv.RouteDeleteNamespaceRulesConfig))
|
||||
group.Delete(toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}"), routing.Wrap(srv.RouteDeleteRuleGroupConfig))
|
||||
group.Get(toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"), routing.Wrap(srv.RouteGetNamespaceRulesConfig))
|
||||
group.Get(toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}"), routing.Wrap(srv.RouteGetRulegGroupConfig))
|
||||
group.Get(toMacaronPath("/api/ruler/{Recipient}/api/v1/rules"), routing.Wrap(srv.RouteGetRulesConfig))
|
||||
group.Post(toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"), binding.Bind(apimodels.PostableRuleGroupConfig{}), routing.Wrap(srv.RoutePostNameRulesConfig))
|
||||
group.Delete(
|
||||
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"),
|
||||
Instrument(
|
||||
http.MethodDelete,
|
||||
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}",
|
||||
srv.RouteDeleteNamespaceRulesConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Delete(
|
||||
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}"),
|
||||
Instrument(
|
||||
http.MethodDelete,
|
||||
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}",
|
||||
srv.RouteDeleteRuleGroupConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}",
|
||||
srv.RouteGetNamespaceRulesConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}/{Groupname}",
|
||||
srv.RouteGetRulegGroupConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Get(
|
||||
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules"),
|
||||
Instrument(
|
||||
http.MethodGet,
|
||||
"/api/ruler/{Recipient}/api/v1/rules",
|
||||
srv.RouteGetRulesConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Post(
|
||||
toMacaronPath("/api/ruler/{Recipient}/api/v1/rules/{Namespace}"),
|
||||
binding.Bind(apimodels.PostableRuleGroupConfig{}),
|
||||
Instrument(
|
||||
http.MethodPost,
|
||||
"/api/ruler/{Recipient}/api/v1/rules/{Namespace}",
|
||||
srv.RoutePostNameRulesConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
}, middleware.ReqSignedIn)
|
||||
}
|
||||
|
||||
@ -8,6 +8,8 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/go-macaron/binding"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
@ -23,10 +25,37 @@ type TestingApiService interface {
|
||||
RouteTestRuleConfig(*models.ReqContext, apimodels.TestRulePayload) response.Response
|
||||
}
|
||||
|
||||
func (api *API) RegisterTestingApiEndpoints(srv TestingApiService) {
|
||||
func (api *API) RegisterTestingApiEndpoints(srv TestingApiService, metrics *Metrics) {
|
||||
api.RouteRegister.Group("", func(group routing.RouteRegister) {
|
||||
group.Post(toMacaronPath("/api/v1/eval"), binding.Bind(apimodels.EvalQueriesPayload{}), routing.Wrap(srv.RouteEvalQueries))
|
||||
group.Post(toMacaronPath("/api/v1/receiver/test/{Recipient}"), binding.Bind(apimodels.ExtendedReceiver{}), routing.Wrap(srv.RouteTestReceiverConfig))
|
||||
group.Post(toMacaronPath("/api/v1/rule/test/{Recipient}"), binding.Bind(apimodels.TestRulePayload{}), routing.Wrap(srv.RouteTestRuleConfig))
|
||||
group.Post(
|
||||
toMacaronPath("/api/v1/eval"),
|
||||
binding.Bind(apimodels.EvalQueriesPayload{}),
|
||||
Instrument(
|
||||
http.MethodPost,
|
||||
"/api/v1/eval",
|
||||
srv.RouteEvalQueries,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Post(
|
||||
toMacaronPath("/api/v1/receiver/test/{Recipient}"),
|
||||
binding.Bind(apimodels.ExtendedReceiver{}),
|
||||
Instrument(
|
||||
http.MethodPost,
|
||||
"/api/v1/receiver/test/{Recipient}",
|
||||
srv.RouteTestReceiverConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
group.Post(
|
||||
toMacaronPath("/api/v1/rule/test/{Recipient}"),
|
||||
binding.Bind(apimodels.TestRulePayload{}),
|
||||
Instrument(
|
||||
http.MethodPost,
|
||||
"/api/v1/rule/test/{Recipient}",
|
||||
srv.RouteTestRuleConfig,
|
||||
metrics,
|
||||
),
|
||||
)
|
||||
}, middleware.ReqSignedIn)
|
||||
}
|
||||
|
||||
149
pkg/services/ngalert/api/metrics.go
Normal file
149
pkg/services/ngalert/api/metrics.go
Normal file
@ -0,0 +1,149 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/grafana/grafana/pkg/api/response"
|
||||
"github.com/grafana/grafana/pkg/api/routing"
|
||||
"github.com/grafana/grafana/pkg/models"
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
"github.com/prometheus/client_golang/prometheus/promauto"
|
||||
"gopkg.in/macaron.v1"
|
||||
)
|
||||
|
||||
const (
|
||||
GrafanaBackend = "grafana"
|
||||
ProxyBackend = "proxy"
|
||||
)
|
||||
|
||||
type Metrics struct {
|
||||
alerts *prometheus.GaugeVec
|
||||
alertsInvalid prometheus.Counter
|
||||
alertsReceived prometheus.Counter
|
||||
notificationLatency prometheus.Histogram
|
||||
notifications *prometheus.CounterVec
|
||||
notificationsFailed *prometheus.CounterVec
|
||||
requestDuration *prometheus.HistogramVec
|
||||
silences *prometheus.GaugeVec
|
||||
}
|
||||
|
||||
func NewMetrics(r prometheus.Registerer) *Metrics {
|
||||
return &Metrics{
|
||||
alerts: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: "grafana",
|
||||
Subsystem: "alerting",
|
||||
Name: "alerts",
|
||||
Help: "How many alerts by state.",
|
||||
}, []string{"state"}),
|
||||
alertsInvalid: promauto.With(r).NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "grafana",
|
||||
Subsystem: "alerting",
|
||||
Name: "alerts_invalid_total",
|
||||
Help: "The total number of invalid received alerts.",
|
||||
}),
|
||||
alertsReceived: promauto.With(r).NewCounter(prometheus.CounterOpts{
|
||||
Namespace: "grafana",
|
||||
Subsystem: "alerting",
|
||||
Name: "alerts_received_total",
|
||||
Help: "The total number of received alerts.",
|
||||
}),
|
||||
notificationLatency: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: "grafana",
|
||||
Subsystem: "alerting",
|
||||
Name: "notification_latency_seconds",
|
||||
Help: "Histogram of notification deliveries",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
}),
|
||||
notifications: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "grafana",
|
||||
Subsystem: "alerting",
|
||||
Name: "notifications_total",
|
||||
Help: "The total number of attempted notfications by integration.",
|
||||
}, []string{"integration"}),
|
||||
notificationsFailed: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
|
||||
Namespace: "grafana",
|
||||
Subsystem: "alerting",
|
||||
Name: "notifications_failed_total",
|
||||
Help: "The total number of failed notfications by integration.",
|
||||
}, []string{"integration"}),
|
||||
requestDuration: promauto.With(r).NewHistogramVec(
|
||||
prometheus.HistogramOpts{
|
||||
Namespace: "grafana",
|
||||
Subsystem: "alerting",
|
||||
Name: "request_duration_seconds",
|
||||
Help: "Histogram of requests to the Alerting API",
|
||||
Buckets: prometheus.DefBuckets,
|
||||
},
|
||||
[]string{"method", "route", "status_code", "backend"},
|
||||
),
|
||||
silences: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
|
||||
Namespace: "grafana",
|
||||
Subsystem: "alerting",
|
||||
Name: "silences",
|
||||
Help: "The total number of silences by state.",
|
||||
}, []string{"state"}),
|
||||
}
|
||||
}
|
||||
|
||||
// Instrument wraps a middleware, instrumenting the request latencies.
|
||||
func Instrument(
|
||||
method,
|
||||
path string,
|
||||
action interface{},
|
||||
metrics *Metrics,
|
||||
) macaron.Handler {
|
||||
normalizedPath := MakeLabelValue(path)
|
||||
|
||||
return func(c *models.ReqContext) {
|
||||
start := time.Now()
|
||||
var res response.Response
|
||||
val, err := c.Invoke(action)
|
||||
if err == nil && val != nil && len(val) > 0 {
|
||||
res = val[0].Interface().(response.Response)
|
||||
} else {
|
||||
res = routing.ServerError(err)
|
||||
}
|
||||
|
||||
// TODO: We could look up the datasource type via our datasource service
|
||||
var backend string
|
||||
recipient := c.Params("Recipient")
|
||||
if recipient == apimodels.GrafanaBackend.String() || recipient == "" {
|
||||
backend = GrafanaBackend
|
||||
} else {
|
||||
backend = ProxyBackend
|
||||
}
|
||||
|
||||
ls := prometheus.Labels{
|
||||
"method": method,
|
||||
"route": normalizedPath,
|
||||
"status_code": fmt.Sprint(res.Status()),
|
||||
"backend": backend,
|
||||
}
|
||||
res.WriteTo(c)
|
||||
metrics.requestDuration.With(ls).Observe(time.Since(start).Seconds())
|
||||
}
|
||||
}
|
||||
|
||||
var invalidChars = regexp.MustCompile(`[^a-zA-Z0-9]+`)
|
||||
|
||||
// MakeLabelValue normalizes a path template
|
||||
func MakeLabelValue(path string) string {
|
||||
// Convert non-alnums to underscores.
|
||||
result := invalidChars.ReplaceAllString(path, "_")
|
||||
|
||||
// Trim leading and trailing underscores.
|
||||
result = strings.Trim(result, "_")
|
||||
|
||||
// Make it all lowercase
|
||||
result = strings.ToLower(result)
|
||||
|
||||
// Special case.
|
||||
if result == "" {
|
||||
result = "root"
|
||||
}
|
||||
return result
|
||||
}
|
||||
@ -16,9 +16,18 @@ type {{classname}}Service interface { {{#operation}}
|
||||
{{nickname}}(*models.ReqContext{{#bodyParams}}, apimodels.{{dataType}}{{/bodyParams}}) response.Response{{/operation}}
|
||||
}
|
||||
|
||||
func (api *API) Register{{classname}}Endpoints(srv {{classname}}Service) {
|
||||
func (api *API) Register{{classname}}Endpoints(srv {{classname}}Service, metrics *Metrics) {
|
||||
api.RouteRegister.Group("", func(group routing.RouteRegister){ {{#operations}}{{#operation}}
|
||||
group.{{httpMethod}}(toMacaronPath("{{{path}}}"){{#bodyParams}}, binding.Bind(apimodels.{{dataType}}{}){{/bodyParams}}, routing.Wrap(srv.{{nickname}})){{/operation}}{{/operations}}
|
||||
group.{{httpMethod}}(
|
||||
toMacaronPath("{{{path}}}"){{#bodyParams}},
|
||||
binding.Bind(apimodels.{{dataType}}{}){{/bodyParams}},
|
||||
Instrument(
|
||||
http.Method{{httpMethod}},
|
||||
"{{{path}}}",
|
||||
srv.{{nickname}},
|
||||
metrics,
|
||||
),
|
||||
){{/operation}}{{/operations}}
|
||||
}, middleware.ReqSignedIn)
|
||||
}{{#operation}}
|
||||
{{/operation}}{{/operations}}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user