Metrics: Expose functions to initialize counters at zero (#50122)

pkg/infra/metrics/metrics.go

@@ -3,9 +3,9 @@ package metrics
 import (
 	"runtime"
 
-	"github.com/prometheus/client_golang/prometheus"
+	"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
-
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // ExporterName is used as namespace for exposing prometheus metrics
@@ -199,40 +199,40 @@ func init() {
 		Namespace: ExporterName,
 	})
 
-	MPageStatus = newCounterVecStartingAtZero(
+	MPageStatus = metricutil.NewCounterVecStartingAtZero(
 		prometheus.CounterOpts{
 			Name:      "page_response_status_total",
 			Help:      "page http response status",
 			Namespace: ExporterName,
-		}, []string{"code"}, httpStatusCodes...)
+		}, []string{"code"}, map[string][]string{"code": httpStatusCodes})
 
-	MApiStatus = newCounterVecStartingAtZero(
+	MApiStatus = metricutil.NewCounterVecStartingAtZero(
 		prometheus.CounterOpts{
 			Name:      "api_response_status_total",
 			Help:      "api http response status",
 			Namespace: ExporterName,
-		}, []string{"code"}, httpStatusCodes...)
+		}, []string{"code"}, map[string][]string{"code": httpStatusCodes})
 
-	MProxyStatus = newCounterVecStartingAtZero(
+	MProxyStatus = metricutil.NewCounterVecStartingAtZero(
 		prometheus.CounterOpts{
 			Name:      "proxy_response_status_total",
 			Help:      "proxy http response status",
 			Namespace: ExporterName,
-		}, []string{"code"}, httpStatusCodes...)
+		}, []string{"code"}, map[string][]string{"code": httpStatusCodes})
 
-	MApiUserSignUpStarted = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiUserSignUpStarted = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_user_signup_started_total",
 		Help:      "amount of users who started the signup flow",
 		Namespace: ExporterName,
 	})
 
-	MApiUserSignUpCompleted = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiUserSignUpCompleted = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_user_signup_completed_total",
 		Help:      "amount of users who completed the signup flow",
 		Namespace: ExporterName,
 	})
 
-	MApiUserSignUpInvite = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiUserSignUpInvite = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_user_signup_invite_total",
 		Help:      "amount of users who have been invited",
 		Namespace: ExporterName,
@@ -259,55 +259,55 @@ func init() {
 		Namespace:  ExporterName,
 	})
 
-	MApiAdminUserCreate = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiAdminUserCreate = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_admin_user_created_total",
 		Help:      "api admin user created counter",
 		Namespace: ExporterName,
 	})
 
-	MApiLoginPost = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiLoginPost = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_login_post_total",
 		Help:      "api login post counter",
 		Namespace: ExporterName,
 	})
 
-	MApiLoginOAuth = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiLoginOAuth = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_login_oauth_total",
 		Help:      "api login oauth counter",
 		Namespace: ExporterName,
 	})
 
-	MApiLoginSAML = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiLoginSAML = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_login_saml_total",
 		Help:      "api login saml counter",
 		Namespace: ExporterName,
 	})
 
-	MApiOrgCreate = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiOrgCreate = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_org_create_total",
 		Help:      "api org created counter",
 		Namespace: ExporterName,
 	})
 
-	MApiDashboardSnapshotCreate = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiDashboardSnapshotCreate = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_dashboard_snapshot_create_total",
 		Help:      "dashboard snapshots created",
 		Namespace: ExporterName,
 	})
 
-	MApiDashboardSnapshotExternal = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiDashboardSnapshotExternal = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_dashboard_snapshot_external_total",
 		Help:      "external dashboard snapshots created",
 		Namespace: ExporterName,
 	})
 
-	MApiDashboardSnapshotGet = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiDashboardSnapshotGet = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_dashboard_snapshot_get_total",
 		Help:      "loaded dashboards",
 		Namespace: ExporterName,
 	})
 
-	MApiDashboardInsert = newCounterStartingAtZero(prometheus.CounterOpts{
+	MApiDashboardInsert = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "api_models_dashboard_insert_total",
 		Help:      "dashboards inserted ",
 		Namespace: ExporterName,
@@ -331,25 +331,25 @@ func init() {
 		Namespace: ExporterName,
 	}, []string{"type"})
 
-	MAwsCloudWatchGetMetricStatistics = newCounterStartingAtZero(prometheus.CounterOpts{
+	MAwsCloudWatchGetMetricStatistics = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "aws_cloudwatch_get_metric_statistics_total",
 		Help:      "counter for getting metric statistics from aws",
 		Namespace: ExporterName,
 	})
 
-	MAwsCloudWatchListMetrics = newCounterStartingAtZero(prometheus.CounterOpts{
+	MAwsCloudWatchListMetrics = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "aws_cloudwatch_list_metrics_total",
 		Help:      "counter for getting list of metrics from aws",
 		Namespace: ExporterName,
 	})
 
-	MAwsCloudWatchGetMetricData = newCounterStartingAtZero(prometheus.CounterOpts{
+	MAwsCloudWatchGetMetricData = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "aws_cloudwatch_get_metric_data_total",
 		Help:      "counter for getting metric data time series from aws",
 		Namespace: ExporterName,
 	})
 
-	MDBDataSourceQueryByID = newCounterStartingAtZero(prometheus.CounterOpts{
+	MDBDataSourceQueryByID = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{
 		Name:      "db_datasource_query_by_id_total",
 		Help:      "counter for getting datasource by id",
 		Namespace: ExporterName,
@@ -646,19 +646,3 @@ func initMetricVars() {
 		StatsTotalDataKeys,
 	)
 }
-
-func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec {
-	counter := prometheus.NewCounterVec(opts, labels)
-
-	for _, label := range labelValues {
-		counter.WithLabelValues(label).Add(0)
-	}
-
-	return counter
-}
-
-func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter {
-	counter := prometheus.NewCounter(opts)
-	counter.Add(0)
-	return counter
-}

pkg/infra/metrics/metricutil/utils.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // SanitizeLabelName removes all invalid chars from the label name.
@@ -29,3 +31,56 @@ func SanitizeLabelName(name string) (string, error) {
 
 	return out.String(), nil
 }
+
+// NewCounterStartingAtZero initializes a new Prometheus counter with an initial
+// observation of zero. Used for to guarantee the existence of the specific metric.
+func NewCounterStartingAtZero(opts prometheus.CounterOpts) prometheus.Counter {
+	counter := prometheus.NewCounter(opts)
+	counter.Add(0)
+	return counter
+}
+
+// NewCounterVecStartingAtZero initializes a new Prometheus counter with an initial
+// observation of zero for every possible value of each label. Used for the sake of
+// consistency among all the possible labels and values.
+func NewCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues map[string][]string) *prometheus.CounterVec {
+	counter := prometheus.NewCounterVec(opts, labels)
+
+	for _, ls := range buildLabelSets(labels, labelValues) {
+		counter.With(ls).Add(0)
+	}
+
+	return counter
+}
+
+func buildLabelSets(labels []string, labelValues map[string][]string) []prometheus.Labels {
+	var labelSets []prometheus.Labels
+
+	var n func(i int, ls prometheus.Labels)
+	n = func(i int, ls prometheus.Labels) {
+		if i == len(labels) {
+			labelSets = append(labelSets, ls)
+			return
+		}
+
+		label := labels[i]
+		values := labelValues[label]
+
+		for _, v := range values {
+			lsCopy := copyLabelSet(ls)
+			lsCopy[label] = v
+			n(i+1, lsCopy)
+		}
+	}
+
+	n(0, prometheus.Labels{})
+	return labelSets
+}
+
+func copyLabelSet(ls prometheus.Labels) prometheus.Labels {
+	newLs := make(prometheus.Labels, len(ls))
+	for l, v := range ls {
+		newLs[l] = v
+	}
+	return newLs
+}

pkg/infra/metrics/metricutil/utils_test.go

@@ -3,6 +3,7 @@ package metricutil
 import (
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -30,3 +31,89 @@ func TestLabelNameSanitization(t *testing.T) {
 		}
 	}
 }
+
+func Test_buildLabelSets(t *testing.T) {
+	testcases := map[string]struct {
+		labels      []string
+		labelValues map[string][]string
+		expected    []prometheus.Labels
+	}{
+		"single label, single value": {
+			labels: []string{"operation"},
+			labelValues: map[string][]string{
+				"operation": {"insert"},
+			},
+			expected: []prometheus.Labels{
+				map[string]string{"operation": "insert"},
+			},
+		},
+		"single label, multiple values": {
+			labels: []string{"operation"},
+			labelValues: map[string][]string{
+				"operation": {"insert", "delete"},
+			},
+			expected: []prometheus.Labels{
+				map[string]string{"operation": "insert"},
+				map[string]string{"operation": "delete"},
+			},
+		},
+		"multiple label, single value": {
+			labels: []string{"operation", "success"},
+			labelValues: map[string][]string{
+				"operation": {"insert"},
+				"success":   {"true"},
+			},
+			expected: []prometheus.Labels{
+				map[string]string{"operation": "insert", "success": "true"},
+			},
+		},
+		"multiple label, multiple values": {
+			labels: []string{"operation", "success"},
+			labelValues: map[string][]string{
+				"operation": {"insert", "delete"},
+				"success":   {"true", "false"},
+			},
+			expected: []prometheus.Labels{
+				map[string]string{"operation": "insert", "success": "true"},
+				map[string]string{"operation": "insert", "success": "false"},
+				map[string]string{"operation": "delete", "success": "true"},
+				map[string]string{"operation": "delete", "success": "false"},
+			},
+		},
+		"irregular labels and values": {
+			labels: []string{"operation", "success", "environment"},
+			labelValues: map[string][]string{
+				"operation":   {"insert", "update", "delete"},
+				"success":     {"true", "false"},
+				"environment": {"dev", "test", "staging"},
+			},
+			expected: []prometheus.Labels{
+				map[string]string{"operation": "insert", "success": "true", "environment": "dev"},
+				map[string]string{"operation": "insert", "success": "true", "environment": "test"},
+				map[string]string{"operation": "insert", "success": "true", "environment": "staging"},
+				map[string]string{"operation": "insert", "success": "false", "environment": "dev"},
+				map[string]string{"operation": "insert", "success": "false", "environment": "test"},
+				map[string]string{"operation": "insert", "success": "false", "environment": "staging"},
+				map[string]string{"operation": "update", "success": "true", "environment": "dev"},
+				map[string]string{"operation": "update", "success": "true", "environment": "test"},
+				map[string]string{"operation": "update", "success": "true", "environment": "staging"},
+				map[string]string{"operation": "update", "success": "false", "environment": "dev"},
+				map[string]string{"operation": "update", "success": "false", "environment": "test"},
+				map[string]string{"operation": "update", "success": "false", "environment": "staging"},
+				map[string]string{"operation": "delete", "success": "true", "environment": "dev"},
+				map[string]string{"operation": "delete", "success": "true", "environment": "test"},
+				map[string]string{"operation": "delete", "success": "true", "environment": "staging"},
+				map[string]string{"operation": "delete", "success": "false", "environment": "dev"},
+				map[string]string{"operation": "delete", "success": "false", "environment": "test"},
+				map[string]string{"operation": "delete", "success": "false", "environment": "staging"},
+			},
+		},
+	}
+
+	for name, tc := range testcases {
+		t.Run(name, func(t *testing.T) {
+			got := buildLabelSets(tc.labels, tc.labelValues)
+			assert.Equal(t, tc.expected, got)
+		})
+	}
+}

pkg/services/secrets/manager/metrics.go

@@ -2,6 +2,7 @@ package manager
 
 import (
 	"github.com/grafana/grafana/pkg/infra/metrics"
+	"github.com/grafana/grafana/pkg/infra/metrics/metricutil"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -11,21 +12,29 @@ const (
 )
 
 var (
-	opsCounter = prometheus.NewCounterVec(
+	opsCounter = metricutil.NewCounterVecStartingAtZero(
 		prometheus.CounterOpts{
 			Namespace: metrics.ExporterName,
 			Name:      "encryption_ops_total",
 			Help:      "A counter for encryption operations",
 		},
 		[]string{"success", "operation"},
+		map[string][]string{
+			"success":   {"true", "false"},
+			"operation": {OpEncrypt, OpDecrypt},
+		},
 	)
-	cacheReadsCounter = prometheus.NewCounterVec(
+	cacheReadsCounter = metricutil.NewCounterVecStartingAtZero(
 		prometheus.CounterOpts{
 			Namespace: metrics.ExporterName,
 			Name:      "encryption_cache_reads_total",
 			Help:      "A counter for encryption cache reads",
 		},
 		[]string{"hit", "method"},
+		map[string][]string{
+			"hit":    {"true", "false"},
+			"method": {"byId", "byName"},
+		},
 	)
 )