Access Control: Add histograms for evaluator and permissions checks (#34026)

This patch adds metrics to support instrumenting the accesscontrols package.
It also instruments the accesscontrol evaluator and the permissions function.

Co-authored-by: Vardan Torosyan <vardants@gmail.com>

pkg/infra/metrics/metrics.go

@@ -12,6 +12,7 @@ import (
 const ExporterName = "grafana"
 
 var (
+
 	// MInstanceStart is a metric counter for started instances
 	MInstanceStart prometheus.Counter
 
@@ -104,6 +105,9 @@ var (
 
 	// MRenderingQueue is a metric gauge for image rendering queue size
 	MRenderingQueue prometheus.Gauge
+
+	// MAccessEvaluationCount is a metric gauge for total number of evaluation requests
+	MAccessEvaluationCount prometheus.Counter
 )
 
 // Timers
@@ -116,6 +120,12 @@ var (
 
 	// MRenderingSummary is a metric summary for image rendering request duration
 	MRenderingSummary *prometheus.SummaryVec
+
+	// MAccessPermissionsSummary is a metric summary for loading permissions request duration when evaluating access
+	MAccessPermissionsSummary prometheus.Histogram
+
+	// MAccessEvaluationsSummary is a metric summary for loading permissions request duration when evaluating access
+	MAccessEvaluationsSummary prometheus.Histogram
 )
 
 // StatTotals
@@ -510,6 +520,24 @@ func init() {
 		Help:      "total amount of annotations in the database",
 		Namespace: ExporterName,
 	})
+
+	MAccessPermissionsSummary = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "access_permissions_duration",
+		Help:    "Histogram for the runtime of permissions check function.",
+		Buckets: prometheus.ExponentialBuckets(0.00001, 4, 10),
+	})
+
+	MAccessEvaluationsSummary = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name:    "access_evaluation_duration",
+		Help:    "Histogram for the runtime of evaluation function.",
+		Buckets: prometheus.ExponentialBuckets(0.00001, 4, 10),
+	})
+
+	MAccessEvaluationCount = prometheus.NewCounter(prometheus.CounterOpts{
+		Name:      "access_evaluation_count",
+		Help:      "number of evaluation calls",
+		Namespace: ExporterName,
+	})
 }
 
 // SetBuildInformation sets the build information for this binary
@@ -582,6 +610,8 @@ func initMetricVars() {
 		MRenderingRequestTotal,
 		MRenderingSummary,
 		MRenderingQueue,
+		MAccessPermissionsSummary,
+		MAccessEvaluationsSummary,
 		MAlertingActiveAlerts,
 		MStatTotalDashboards,
 		MStatTotalFolders,
@@ -600,6 +630,7 @@ func initMetricVars() {
 		grafanaPluginBuildInfoDesc,
 		StatsTotalDashboardVersions,
 		StatsTotalAnnotations,
+		MAccessEvaluationCount,
 	)
 }
 
@@ -616,6 +647,5 @@ func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, l
 func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter {
 	counter := prometheus.NewCounter(opts)
 	counter.Add(0)
-
 	return counter
 }

pkg/services/accesscontrol/evaluator/evaluator.go

@@ -5,13 +5,18 @@ import (
 
 	"github.com/gobwas/glob"
 
+	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // Evaluate evaluates access to the given resource, using provided AccessControl instance.
 // Scopes are evaluated with an `OR` relationship.
 func Evaluate(ctx context.Context, ac accesscontrol.AccessControl, user *models.SignedInUser, action string, scope ...string) (bool, error) {
+	timer := prometheus.NewTimer(metrics.MAccessEvaluationsSummary)
+	defer timer.ObserveDuration()
+	metrics.MAccessEvaluationCount.Inc()
 	userPermissions, err := ac.GetUserPermissions(ctx, user)
 	if err != nil {
 		return false, err
@@ -22,7 +27,8 @@ func Evaluate(ctx context.Context, ac accesscontrol.AccessControl, user *models.
 		return false, nil
 	}
 
-	return evaluateScope(dbScopes, scope...)
+	res, err := evaluateScope(dbScopes, scope...)
+	return res, err
 }
 
 func evaluateScope(dbScopes map[string]struct{}, targetScopes ...string) (bool, error) {

pkg/services/accesscontrol/ossaccesscontrol/ossaccesscontrol.go

@@ -4,10 +4,12 @@ import (
 	"context"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/accesscontrol/evaluator"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // OSSAccessControlService is the service implementing role based access control.
@@ -39,6 +41,9 @@ func (ac *OSSAccessControlService) Evaluate(ctx context.Context, user *models.Si
 
 // GetUserPermissions returns user permissions based on built-in roles
 func (ac *OSSAccessControlService) GetUserPermissions(ctx context.Context, user *models.SignedInUser) ([]*accesscontrol.Permission, error) {
+	timer := prometheus.NewTimer(metrics.MAccessPermissionsSummary)
+	defer timer.ObserveDuration()
+
 	builtinRoles := ac.GetUserBuiltInRoles(user)
 	permissions := make([]*accesscontrol.Permission, 0)
 	for _, builtin := range builtinRoles {