Zanzana: Add basic metrics for evaluation time and status (#90125)

* Zanzana: Add basic metrics for evaluation time and status

* only init when feature enabled

* add todo

pkg/services/accesscontrol/acimpl/accesscontrol.go

@@ -24,8 +24,18 @@ var _ accesscontrol.AccessControl = new(AccessControl)
 
 func ProvideAccessControl(features featuremgmt.FeatureToggles, zclient zanzana.Client) *AccessControl {
 	logger := log.New("accesscontrol")
+
+	var m *acMetrics
+	if features.IsEnabledGlobally(featuremgmt.FlagZanzana) {
+		m = initMetrics()
+	}
+
 	return &AccessControl{
-		features, logger, accesscontrol.NewResolvers(logger), zclient,
+		features,
+		logger,
+		accesscontrol.NewResolvers(logger),
+		zclient,
+		m,
 	}
 }
 
@@ -38,6 +48,7 @@ type AccessControl struct {
 	log       log.Logger
 	resolvers accesscontrol.Resolvers
 	zclient   zanzana.Client
+	metrics   *acMetrics
 }
 
 func (a *AccessControl) Evaluate(ctx context.Context, user identity.Requester, evaluator accesscontrol.Evaluator) (bool, error) {
@@ -130,13 +141,19 @@ type evalResult struct {
 func (a *AccessControl) evaluateCompare(ctx context.Context, user identity.Requester, evaluator accesscontrol.Evaluator) (bool, error) {
 	res := make(chan evalResult, 2)
 	go func() {
+		timer := prometheus.NewTimer(a.metrics.mAccessEngineEvaluationsSeconds.WithLabelValues("zanzana"))
+		defer timer.ObserveDuration()
 		start := time.Now()
+
 		hasAccess, err := a.evaluateZanzana(ctx, user, evaluator)
 		res <- evalResult{"zanzana", hasAccess, err, time.Since(start)}
 	}()
 
 	go func() {
+		timer := prometheus.NewTimer(a.metrics.mAccessEngineEvaluationsSeconds.WithLabelValues("grafana"))
+		defer timer.ObserveDuration()
 		start := time.Now()
+
 		hasAccess, err := a.evaluate(ctx, user, evaluator)
 		res <- evalResult{"grafana", hasAccess, err, time.Since(start)}
 	}()
@@ -151,6 +168,7 @@ func (a *AccessControl) evaluateCompare(ctx context.Context, user identity.Reque
 		if second.err != nil {
 			a.log.Error("zanzana evaluation failed", "error", second.err)
 		} else if first.decision != second.decision {
+			a.metrics.mZanzanaEvaluationStatusTotal.WithLabelValues("error").Inc()
 			a.log.Warn(
 				"zanzana evaluation result does not match grafana",
 				"grafana_decision", first.decision,
@@ -160,6 +178,7 @@ func (a *AccessControl) evaluateCompare(ctx context.Context, user identity.Reque
 				"eval", evaluator.GoString(),
 			)
 		} else {
+			a.metrics.mZanzanaEvaluationStatusTotal.WithLabelValues("success").Inc()
 			a.log.Debug("zanzana evaluation is correct", "grafana_ms", first.duration, "zanzana_ms", second.duration)
 		}
 	}

pkg/services/accesscontrol/acimpl/metrics.go

@@ -0,0 +1,53 @@
+package acimpl
+
+import (
+	"sync"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
+)
+
+const (
+	metricsSubSystem = "authz"
+	metricsNamespace = "grafana"
+)
+
+type acMetrics struct {
+	// mAccessEngineEvaluationsSeconds is a summary for evaluating access for a specific engine (RBAC and zanzana)
+	mAccessEngineEvaluationsSeconds *prometheus.HistogramVec
+	// mZanzanaEvaluationStatusTotal is a metric for zanzana evaluation status
+	mZanzanaEvaluationStatusTotal *prometheus.CounterVec
+}
+
+var once sync.Once
+
+// TODO: use prometheus.Registerer
+func initMetrics() *acMetrics {
+	m := &acMetrics{}
+	once.Do(func() {
+		m.mAccessEngineEvaluationsSeconds = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Name:      "engine_evaluations_seconds",
+			Help:      "Histogram for evaluation time for the specific access control engine (RBAC and zanzana).",
+			Namespace: metricsNamespace,
+			Subsystem: metricsSubSystem,
+			Buckets:   prometheus.ExponentialBuckets(0.00001, 4, 10),
+		},
+			[]string{"engine"},
+		)
+
+		m.mZanzanaEvaluationStatusTotal = metricutil.NewCounterVecStartingAtZero(
+			prometheus.CounterOpts{
+				Name:      "zanzana_evaluation_status_total",
+				Help:      "evaluation status (success or error) for zanzana",
+				Namespace: metricsNamespace,
+				Subsystem: metricsSubSystem,
+			}, []string{"status"}, map[string][]string{"status": {"success", "error"}})
+
+		prometheus.MustRegister(
+			m.mAccessEngineEvaluationsSeconds,
+			m.mZanzanaEvaluationStatusTotal,
+		)
+	})
+	return m
+}