From 97baa6911d68c984f8e18a58fb014bbbdf05d8e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Joan=20L=C3=B3pez=20de=20la=20Franca=20Beltran?= <5459617+joanlopez@users.noreply.github.com> Date: Mon, 13 Jun 2022 17:35:10 +0200 Subject: [PATCH] Metrics: Expose functions to initialize counters at zero (#50122) --- pkg/infra/metrics/metrics.go | 64 ++++++---------- pkg/infra/metrics/metricutil/utils.go | 55 ++++++++++++++ pkg/infra/metrics/metricutil/utils_test.go | 87 ++++++++++++++++++++++ pkg/services/secrets/manager/metrics.go | 13 +++- 4 files changed, 177 insertions(+), 42 deletions(-) diff --git a/pkg/infra/metrics/metrics.go b/pkg/infra/metrics/metrics.go index 3eac92d8987..85d4e628297 100644 --- a/pkg/infra/metrics/metrics.go +++ b/pkg/infra/metrics/metrics.go @@ -3,9 +3,9 @@ package metrics import ( "runtime" - "github.com/prometheus/client_golang/prometheus" - + "github.com/grafana/grafana/pkg/infra/metrics/metricutil" "github.com/grafana/grafana/pkg/setting" + "github.com/prometheus/client_golang/prometheus" ) // ExporterName is used as namespace for exposing prometheus metrics @@ -199,40 +199,40 @@ func init() { Namespace: ExporterName, }) - MPageStatus = newCounterVecStartingAtZero( + MPageStatus = metricutil.NewCounterVecStartingAtZero( prometheus.CounterOpts{ Name: "page_response_status_total", Help: "page http response status", Namespace: ExporterName, - }, []string{"code"}, httpStatusCodes...) + }, []string{"code"}, map[string][]string{"code": httpStatusCodes}) - MApiStatus = newCounterVecStartingAtZero( + MApiStatus = metricutil.NewCounterVecStartingAtZero( prometheus.CounterOpts{ Name: "api_response_status_total", Help: "api http response status", Namespace: ExporterName, - }, []string{"code"}, httpStatusCodes...) + }, []string{"code"}, map[string][]string{"code": httpStatusCodes}) - MProxyStatus = newCounterVecStartingAtZero( + MProxyStatus = metricutil.NewCounterVecStartingAtZero( prometheus.CounterOpts{ Name: "proxy_response_status_total", Help: "proxy http response status", Namespace: ExporterName, - }, []string{"code"}, httpStatusCodes...) + }, []string{"code"}, map[string][]string{"code": httpStatusCodes}) - MApiUserSignUpStarted = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiUserSignUpStarted = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_user_signup_started_total", Help: "amount of users who started the signup flow", Namespace: ExporterName, }) - MApiUserSignUpCompleted = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiUserSignUpCompleted = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_user_signup_completed_total", Help: "amount of users who completed the signup flow", Namespace: ExporterName, }) - MApiUserSignUpInvite = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiUserSignUpInvite = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_user_signup_invite_total", Help: "amount of users who have been invited", Namespace: ExporterName, @@ -259,55 +259,55 @@ func init() { Namespace: ExporterName, }) - MApiAdminUserCreate = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiAdminUserCreate = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_admin_user_created_total", Help: "api admin user created counter", Namespace: ExporterName, }) - MApiLoginPost = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiLoginPost = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_login_post_total", Help: "api login post counter", Namespace: ExporterName, }) - MApiLoginOAuth = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiLoginOAuth = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_login_oauth_total", Help: "api login oauth counter", Namespace: ExporterName, }) - MApiLoginSAML = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiLoginSAML = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_login_saml_total", Help: "api login saml counter", Namespace: ExporterName, }) - MApiOrgCreate = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiOrgCreate = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_org_create_total", Help: "api org created counter", Namespace: ExporterName, }) - MApiDashboardSnapshotCreate = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiDashboardSnapshotCreate = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_dashboard_snapshot_create_total", Help: "dashboard snapshots created", Namespace: ExporterName, }) - MApiDashboardSnapshotExternal = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiDashboardSnapshotExternal = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_dashboard_snapshot_external_total", Help: "external dashboard snapshots created", Namespace: ExporterName, }) - MApiDashboardSnapshotGet = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiDashboardSnapshotGet = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_dashboard_snapshot_get_total", Help: "loaded dashboards", Namespace: ExporterName, }) - MApiDashboardInsert = newCounterStartingAtZero(prometheus.CounterOpts{ + MApiDashboardInsert = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "api_models_dashboard_insert_total", Help: "dashboards inserted ", Namespace: ExporterName, @@ -331,25 +331,25 @@ func init() { Namespace: ExporterName, }, []string{"type"}) - MAwsCloudWatchGetMetricStatistics = newCounterStartingAtZero(prometheus.CounterOpts{ + MAwsCloudWatchGetMetricStatistics = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "aws_cloudwatch_get_metric_statistics_total", Help: "counter for getting metric statistics from aws", Namespace: ExporterName, }) - MAwsCloudWatchListMetrics = newCounterStartingAtZero(prometheus.CounterOpts{ + MAwsCloudWatchListMetrics = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "aws_cloudwatch_list_metrics_total", Help: "counter for getting list of metrics from aws", Namespace: ExporterName, }) - MAwsCloudWatchGetMetricData = newCounterStartingAtZero(prometheus.CounterOpts{ + MAwsCloudWatchGetMetricData = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "aws_cloudwatch_get_metric_data_total", Help: "counter for getting metric data time series from aws", Namespace: ExporterName, }) - MDBDataSourceQueryByID = newCounterStartingAtZero(prometheus.CounterOpts{ + MDBDataSourceQueryByID = metricutil.NewCounterStartingAtZero(prometheus.CounterOpts{ Name: "db_datasource_query_by_id_total", Help: "counter for getting datasource by id", Namespace: ExporterName, @@ -646,19 +646,3 @@ func initMetricVars() { StatsTotalDataKeys, ) } - -func newCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues ...string) *prometheus.CounterVec { - counter := prometheus.NewCounterVec(opts, labels) - - for _, label := range labelValues { - counter.WithLabelValues(label).Add(0) - } - - return counter -} - -func newCounterStartingAtZero(opts prometheus.CounterOpts, labelValues ...string) prometheus.Counter { - counter := prometheus.NewCounter(opts) - counter.Add(0) - return counter -} diff --git a/pkg/infra/metrics/metricutil/utils.go b/pkg/infra/metrics/metricutil/utils.go index 7bb3b471333..c26f5363f8a 100644 --- a/pkg/infra/metrics/metricutil/utils.go +++ b/pkg/infra/metrics/metricutil/utils.go @@ -4,6 +4,8 @@ import ( "errors" "fmt" "strings" + + "github.com/prometheus/client_golang/prometheus" ) // SanitizeLabelName removes all invalid chars from the label name. @@ -29,3 +31,56 @@ func SanitizeLabelName(name string) (string, error) { return out.String(), nil } + +// NewCounterStartingAtZero initializes a new Prometheus counter with an initial +// observation of zero. Used for to guarantee the existence of the specific metric. +func NewCounterStartingAtZero(opts prometheus.CounterOpts) prometheus.Counter { + counter := prometheus.NewCounter(opts) + counter.Add(0) + return counter +} + +// NewCounterVecStartingAtZero initializes a new Prometheus counter with an initial +// observation of zero for every possible value of each label. Used for the sake of +// consistency among all the possible labels and values. +func NewCounterVecStartingAtZero(opts prometheus.CounterOpts, labels []string, labelValues map[string][]string) *prometheus.CounterVec { + counter := prometheus.NewCounterVec(opts, labels) + + for _, ls := range buildLabelSets(labels, labelValues) { + counter.With(ls).Add(0) + } + + return counter +} + +func buildLabelSets(labels []string, labelValues map[string][]string) []prometheus.Labels { + var labelSets []prometheus.Labels + + var n func(i int, ls prometheus.Labels) + n = func(i int, ls prometheus.Labels) { + if i == len(labels) { + labelSets = append(labelSets, ls) + return + } + + label := labels[i] + values := labelValues[label] + + for _, v := range values { + lsCopy := copyLabelSet(ls) + lsCopy[label] = v + n(i+1, lsCopy) + } + } + + n(0, prometheus.Labels{}) + return labelSets +} + +func copyLabelSet(ls prometheus.Labels) prometheus.Labels { + newLs := make(prometheus.Labels, len(ls)) + for l, v := range ls { + newLs[l] = v + } + return newLs +} diff --git a/pkg/infra/metrics/metricutil/utils_test.go b/pkg/infra/metrics/metricutil/utils_test.go index d4862529085..b703b9f6811 100644 --- a/pkg/infra/metrics/metricutil/utils_test.go +++ b/pkg/infra/metrics/metricutil/utils_test.go @@ -3,6 +3,7 @@ package metricutil import ( "testing" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -30,3 +31,89 @@ func TestLabelNameSanitization(t *testing.T) { } } } + +func Test_buildLabelSets(t *testing.T) { + testcases := map[string]struct { + labels []string + labelValues map[string][]string + expected []prometheus.Labels + }{ + "single label, single value": { + labels: []string{"operation"}, + labelValues: map[string][]string{ + "operation": {"insert"}, + }, + expected: []prometheus.Labels{ + map[string]string{"operation": "insert"}, + }, + }, + "single label, multiple values": { + labels: []string{"operation"}, + labelValues: map[string][]string{ + "operation": {"insert", "delete"}, + }, + expected: []prometheus.Labels{ + map[string]string{"operation": "insert"}, + map[string]string{"operation": "delete"}, + }, + }, + "multiple label, single value": { + labels: []string{"operation", "success"}, + labelValues: map[string][]string{ + "operation": {"insert"}, + "success": {"true"}, + }, + expected: []prometheus.Labels{ + map[string]string{"operation": "insert", "success": "true"}, + }, + }, + "multiple label, multiple values": { + labels: []string{"operation", "success"}, + labelValues: map[string][]string{ + "operation": {"insert", "delete"}, + "success": {"true", "false"}, + }, + expected: []prometheus.Labels{ + map[string]string{"operation": "insert", "success": "true"}, + map[string]string{"operation": "insert", "success": "false"}, + map[string]string{"operation": "delete", "success": "true"}, + map[string]string{"operation": "delete", "success": "false"}, + }, + }, + "irregular labels and values": { + labels: []string{"operation", "success", "environment"}, + labelValues: map[string][]string{ + "operation": {"insert", "update", "delete"}, + "success": {"true", "false"}, + "environment": {"dev", "test", "staging"}, + }, + expected: []prometheus.Labels{ + map[string]string{"operation": "insert", "success": "true", "environment": "dev"}, + map[string]string{"operation": "insert", "success": "true", "environment": "test"}, + map[string]string{"operation": "insert", "success": "true", "environment": "staging"}, + map[string]string{"operation": "insert", "success": "false", "environment": "dev"}, + map[string]string{"operation": "insert", "success": "false", "environment": "test"}, + map[string]string{"operation": "insert", "success": "false", "environment": "staging"}, + map[string]string{"operation": "update", "success": "true", "environment": "dev"}, + map[string]string{"operation": "update", "success": "true", "environment": "test"}, + map[string]string{"operation": "update", "success": "true", "environment": "staging"}, + map[string]string{"operation": "update", "success": "false", "environment": "dev"}, + map[string]string{"operation": "update", "success": "false", "environment": "test"}, + map[string]string{"operation": "update", "success": "false", "environment": "staging"}, + map[string]string{"operation": "delete", "success": "true", "environment": "dev"}, + map[string]string{"operation": "delete", "success": "true", "environment": "test"}, + map[string]string{"operation": "delete", "success": "true", "environment": "staging"}, + map[string]string{"operation": "delete", "success": "false", "environment": "dev"}, + map[string]string{"operation": "delete", "success": "false", "environment": "test"}, + map[string]string{"operation": "delete", "success": "false", "environment": "staging"}, + }, + }, + } + + for name, tc := range testcases { + t.Run(name, func(t *testing.T) { + got := buildLabelSets(tc.labels, tc.labelValues) + assert.Equal(t, tc.expected, got) + }) + } +} diff --git a/pkg/services/secrets/manager/metrics.go b/pkg/services/secrets/manager/metrics.go index 8176bc61d5a..b435f71036d 100644 --- a/pkg/services/secrets/manager/metrics.go +++ b/pkg/services/secrets/manager/metrics.go @@ -2,6 +2,7 @@ package manager import ( "github.com/grafana/grafana/pkg/infra/metrics" + "github.com/grafana/grafana/pkg/infra/metrics/metricutil" "github.com/prometheus/client_golang/prometheus" ) @@ -11,21 +12,29 @@ const ( ) var ( - opsCounter = prometheus.NewCounterVec( + opsCounter = metricutil.NewCounterVecStartingAtZero( prometheus.CounterOpts{ Namespace: metrics.ExporterName, Name: "encryption_ops_total", Help: "A counter for encryption operations", }, []string{"success", "operation"}, + map[string][]string{ + "success": {"true", "false"}, + "operation": {OpEncrypt, OpDecrypt}, + }, ) - cacheReadsCounter = prometheus.NewCounterVec( + cacheReadsCounter = metricutil.NewCounterVecStartingAtZero( prometheus.CounterOpts{ Namespace: metrics.ExporterName, Name: "encryption_cache_reads_total", Help: "A counter for encryption cache reads", }, []string{"hit", "method"}, + map[string][]string{ + "hit": {"true", "false"}, + "method": {"byId", "byName"}, + }, ) )