gRPC Server: Instrument requests made to the server. (#71914)

* gRPC Server: Instrument requests made to the server.

Expose metrics from the gRPC server in order to monitor for failed responses
and response latency. Uses code from the already vendored weaveworks/common.

* Review comments.
This commit is contained in:
Steve Simpson 2023-07-19 16:01:54 +02:00 committed by GitHub
parent ebc6288191
commit 2d5300ce3a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 33 additions and 1 deletions

3
go.mod
View File

@ -327,6 +327,8 @@ require (
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/getsentry/sentry-go v0.12.0 // indirect
github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
github.com/gogo/googleapis v1.4.1 // indirect
github.com/gogo/status v1.1.1 // indirect
github.com/google/cel-go v0.12.6 // indirect
github.com/google/gnostic v0.6.9 // indirect
github.com/google/go-querystring v1.1.0 // indirect
@ -356,6 +358,7 @@ require (
github.com/onsi/gomega v1.27.6 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.0.3-0.20220512140940-7b36cea86235 // indirect
github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect
github.com/ory/go-acc v0.2.6 // indirect
github.com/ory/go-convenience v0.1.0 // indirect
github.com/ory/viper v1.7.5 // indirect

5
go.sum
View File

@ -1110,6 +1110,7 @@ github.com/gofrs/uuid v4.4.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRx
github.com/gofrs/uuid/v3 v3.1.2/go.mod h1:xPwMqoocQ1L5G6pXX5BcE7N5jlzn2o19oqAKxwZW/kI=
github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
github.com/gogo/googleapis v1.4.1 h1:1Yx4Myt7BxzvUr5ldGSbwYiZG6t9wGBZ+8/fX3Wvtq0=
github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4=
github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
@ -1120,6 +1121,8 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/gogo/status v1.0.3/go.mod h1:SavQ51ycCLnc7dGyJxp8YAmudx8xqiVrRf+6IXRsugc=
github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM=
github.com/gogo/status v1.1.1 h1:DuHXlSFHNKqTQ+/ACf5Vs6r4X/dH2EgIzR9Vr+H65kg=
github.com/gogo/status v1.1.1/go.mod h1:jpG3dM5QPcqu19Hg8lkUhBFBa3TcLs1DG7+2Jqci7oU=
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
@ -1944,6 +1947,8 @@ github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rm
github.com/opentracing-contrib/go-grpc v0.0.0-20180928155321-4b5a12d3ff02/go.mod h1:JNdpVEzCpXBgIiv4ds+TzhN1hrtxq6ClLrTlT9OQRSc=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w=
github.com/opentracing-contrib/go-stdlib v1.0.0 h1:TBS7YuVotp8myLon4Pv7BtCBzOTo1DeZCld0Z63mW2w=
github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=

View File

@ -8,6 +8,9 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware"
grpcAuth "github.com/grpc-ecosystem/go-grpc-middleware/auth"
"github.com/prometheus/client_golang/prometheus"
"github.com/weaveworks/common/instrument"
"github.com/weaveworks/common/middleware"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
@ -19,6 +22,10 @@ import (
"github.com/grafana/grafana/pkg/setting"
)
var (
grpcRequestDuration *prometheus.HistogramVec
)
type Provider interface {
registry.BackgroundService
GetServer() *grpc.Server
@ -32,12 +39,27 @@ type GPRCServerService struct {
address string
}
func ProvideService(cfg *setting.Cfg, authenticator interceptors.Authenticator, tracer tracing.Tracer) (Provider, error) {
func ProvideService(cfg *setting.Cfg, authenticator interceptors.Authenticator, tracer tracing.Tracer, registerer prometheus.Registerer) (Provider, error) {
s := &GPRCServerService{
cfg: cfg,
logger: log.New("grpc-server"),
}
// Register the metric here instead of an init() function so that we do
// nothing unless the feature is actually enabled.
if grpcRequestDuration == nil {
grpcRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
Namespace: "grafana",
Name: "grpc_request_duration_seconds",
Help: "Time (in seconds) spent serving HTTP requests.",
Buckets: instrument.DefBuckets,
}, []string{"method", "route", "status_code", "ws"})
if err := registerer.Register(grpcRequestDuration); err != nil {
return nil, err
}
}
var opts []grpc.ServerOption
// Default auth is admin token check, but this can be overridden by
@ -48,12 +70,14 @@ func ProvideService(cfg *setting.Cfg, authenticator interceptors.Authenticator,
grpc_middleware.ChainUnaryServer(
grpcAuth.UnaryServerInterceptor(authenticator.Authenticate),
interceptors.TracingUnaryInterceptor(tracer),
middleware.UnaryServerInstrumentInterceptor(grpcRequestDuration),
),
),
grpc.StreamInterceptor(
grpc_middleware.ChainStreamServer(
interceptors.TracingStreamInterceptor(tracer),
grpcAuth.StreamServerInterceptor(authenticator.Authenticate),
middleware.StreamServerInstrumentInterceptor(grpcRequestDuration),
),
),
}...)