CloudMonitoring: Use response unit instead of metric descriptor unit (#32928)

* improve unit detection

* Update docs/sources/datasources/google-cloud-monitoring/_index.md

Co-authored-by: Ursula Kallio <73951760+osg-grafana@users.noreply.github.com>

* goimports

* fix typo

* golint fixes

* Update docs/sources/datasources/google-cloud-monitoring/_index.md

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>

* Update docs/sources/datasources/google-cloud-monitoring/_index.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Update docs/sources/datasources/google-cloud-monitoring/_index.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Update docs/sources/datasources/google-cloud-monitoring/_index.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* Update docs/sources/datasources/google-cloud-monitoring/_index.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

* pr feedback

* Update docs/sources/datasources/google-cloud-monitoring/_index.md

Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

Co-authored-by: Ursula Kallio <73951760+osg-grafana@users.noreply.github.com>
Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>
Co-authored-by: achatterjee-grafana <70489351+achatterjee-grafana@users.noreply.github.com>

docs/sources/datasources/google-cloud-monitoring/_index.md

@@ -8,7 +8,7 @@ weight = 200
 
 # Using Google Cloud Monitoring in Grafana
 
-Grafana ships with built-in support for Google Cloud Monitoring. Just add it as a data source and you are ready to build dashboards for your Google Cloud Monitoring metrics.  Refer to [Add a data source]({{< relref "../add-a-data-source.md" >}}) for instructions on how to add a data source to Grafana. Only users with the organization admin role can add data sources.
+Grafana ships with built-in support for Google Cloud Monitoring. Add it as a data source to build dashboards for your Google Cloud Monitoring metrics. For instructions on how to add a data source, refer to [Add a data source]({{< relref "../add-a-data-source.md" >}}). Only users with the organization admin role can add data sources.
 
 > **Note** Before Grafana v7.1, Google Cloud Monitoring was referred to as Google Stackdriver.
 
@@ -16,11 +16,11 @@ Grafana ships with built-in support for Google Cloud Monitoring. Just add it as
 
 To access Google Cloud Monitoring settings, hover your mouse over the **Configuration** (gear) icon, then click **Data Sources**, and then click the Google Cloud Monitoring data source.
 
-| Name                  | Description                                                                           |
+| Name                  | Description                                                                                                                                                                            |
-| --------------------- | ------------------------------------------------------------------------------------- |
+| --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `Name`                | The data source name. This is how you refer to the data source in panels and queries. |
+| `Name`                | The data source name. This is how you refer to the data source in panels and queries.                                                                                                  |
-| `Default`             | Default data source means that it will be pre-selected for new panels.                |
+| `Default`             | Default data source means that it is pre-selected for new panels.                                                                                                                      |
-| `Service Account Key` | Upload or paste in the Service Account Key file for a GCP Project. Refer to [Using a Google Service Account Key File](#using-a-google-service-account-key-file) for details.|
+| `Service Account Key` | Upload or paste in the Service Account Key file for a GCP Project. For more information, refer to [Using a Google Service Account Key File](#using-a-google-service-account-key-file). |
 
 ## Authentication
 
@@ -93,7 +93,7 @@ To create a metric query, follow these steps:
 1. Choose a metric from the **Metric** dropdown.
 1. Use the plus and minus icons in the filter and group by sections to add/remove filters or group by clauses. This step is optional.
 
-Google Cloud Monitoring metrics can be of different kinds (GAUGE, DELTA, CUMULATIVE) and these kinds have support for different aggregation options (reducers and aligners). The Grafana query editor shows the list of available aggregation methods for a selected metric and sets a default reducer and aligner when you select the metric. Units for the Y-axis are also automatically selected by the query editor.
+Google Cloud Monitoring supports different kinds of metrics like `GAUGE`, `DELTA,` and `CUMULATIVE`. They support different aggregation options, for example, reducers and aligners. The Grafana query editor displays the list of available aggregation methods for a selected metric and sets a default reducer and aligner when you select the metric.
 
 #### Filter
 
@@ -122,7 +122,7 @@ The option is called `cloud monitoring auto` and the defaults are:
 - 5m for time ranges >= 23 hours and < 6 days
 - 1h for time ranges >= 6 days
 
-The other automatic option is `grafana auto`. This will automatically set the group by time depending on the time range chosen and the width of the graph panel. For more information about grafana auto, refer to the [interval variable](http://docs.grafana.org/variables/templates-and-variables/#the-interval-variable).
+The other automatic option is `grafana auto`. This will automatically set the group by time depending on the time range chosen and the width of the time series panel. For more information about grafana auto, refer to the [interval variable](http://docs.grafana.org/variables/templates-and-variables/#the-interval-variable).
 
 It is also possible to choose fixed time intervals to group by, like `1h` or `1d`.
 
@@ -182,6 +182,10 @@ Example Result: `gce_instance - compute.googleapis.com/instance/cpu/usage_time`
 Click on a time series in the panel to see a context menu with a link to View in Metrics Explorer in Google Cloud Console. Clicking that link opens the Metrics Explorer in the Google Cloud Console and runs the query from the Grafana panel there.
 The link navigates the user first to the Google Account Chooser and after successfully selecting an account, the user is redirected to the Metrics Explorer. The provided link is valid for any account, but it only displays the query if your account has access to the GCP project specified in the query.
 
+#### Automatic unit detection
+
+Grafana issues one query to the Cloud Monitoring API per query editor row, and each API response includes a unit. Grafana will attempt to convert the returned unit into a unit that is understood by the Grafana time series panel. If the conversion was successful, then the unit will be displayed on the Y-axis on the panel. If the query editor rows returned different units, then the unit from the last query editor row is used in the time series panel.
+
 ### SLO (Service Level Objective) queries
 
 > **Note:** Available in Grafana v7.0 and later versions.

pkg/tsdb/cloudmonitoring/cloudmonitoring.go

@@ -48,6 +48,7 @@ var (
 		"us":      "µs",
 		"ms":      "ms",
 		"ns":      "ns",
+		"%":       "percent",
 		"percent": "percent",
 		"MiBy":    "mbytes",
 		"By/s":    "Bps",
@@ -157,8 +158,6 @@ func (e *Executor) executeTimeSeriesQuery(ctx context.Context, tsdbQuery plugins
 		return plugins.DataResponse{}, err
 	}
 
-	unit := e.resolvePanelUnitFromQueries(queryExecutors)
-
 	for _, queryExecutor := range queryExecutors {
 		queryRes, resp, executedQueryString, err := queryExecutor.run(ctx, tsdbQuery, e)
 		if err != nil {
@@ -170,43 +169,11 @@ func (e *Executor) executeTimeSeriesQuery(ctx context.Context, tsdbQuery plugins
 		}
 
 		result.Results[queryExecutor.getRefID()] = queryRes
-
-		if len(unit) > 0 {
-			frames, _ := queryRes.Dataframes.Decoded()
-			for i := range frames {
-				if frames[i].Fields[1].Config == nil {
-					frames[i].Fields[1].Config = &data.FieldConfig{}
-				}
-				frames[i].Fields[1].Config.Unit = unit
-			}
-			queryRes.Dataframes = plugins.NewDecodedDataFrames(frames)
-		}
-		result.Results[queryExecutor.getRefID()] = queryRes
 	}
 
 	return result, nil
 }
 
-func (e *Executor) resolvePanelUnitFromQueries(executors []cloudMonitoringQueryExecutor) string {
-	if len(executors) == 0 {
-		return ""
-	}
-	unit := executors[0].getUnit()
-	if len(executors) > 1 {
-		for _, query := range executors[1:] {
-			if query.getUnit() != unit {
-				return ""
-			}
-		}
-	}
-	if len(unit) > 0 {
-		if val, ok := cloudMonitoringUnitMappings[unit]; ok {
-			return val
-		}
-	}
-	return ""
-}
-
 func (e *Executor) buildQueryExecutors(tsdbQuery plugins.DataQuery) ([]cloudMonitoringQueryExecutor, error) {
 	cloudMonitoringQueryExecutors := []cloudMonitoringQueryExecutor{}
 
@@ -282,7 +249,6 @@ func (e *Executor) buildQueryExecutors(tsdbQuery plugins.DataQuery) ([]cloudMoni
 		target = params.Encode()
 		cmtsf.Target = target
 		cmtsf.Params = params
-		cmtsf.Unit = q.MetricQuery.Unit
 
 		if setting.Env == setting.Dev {
 			slog.Debug("CloudMonitoring request", "params", params)
@@ -601,7 +567,7 @@ func unmarshalResponse(res *http.Response) (cloudMonitoringResponse, error) {
 	return data, nil
 }
 
-func addConfigData(frames data.Frames, dl string) data.Frames {
+func addConfigData(frames data.Frames, dl string, unit string) data.Frames {
 	for i := range frames {
 		if frames[i].Fields[1].Config == nil {
 			frames[i].Fields[1].Config = &data.FieldConfig{}
@@ -612,6 +578,11 @@ func addConfigData(frames data.Frames, dl string) data.Frames {
 			URL:         dl,
 		}
 		frames[i].Fields[1].Config.Links = append(frames[i].Fields[1].Config.Links, deepLink)
+		if len(unit) > 0 {
+			if val, ok := cloudMonitoringUnitMappings[unit]; ok {
+				frames[i].Fields[1].Config.Unit = val
+			}
+		}
 	}
 	return frames
 }

pkg/tsdb/cloudmonitoring/cloudmonitoring_test.go

@@ -893,77 +893,34 @@ func TestCloudMonitoring(t *testing.T) {
 				assert.Equal(t, "select_slo_compliance(\"projects/test-proj/services/test-service/serviceLevelObjectives/test-slo\")", frames[0].Fields[1].Name)
 			})
 		})
-	})
 
-	t.Run("Parse cloud monitoring unit", func(t *testing.T) {
+		t.Run("Parse cloud monitoring unit", func(t *testing.T) {
-		t.Run("when there is only one query", func(t *testing.T) {
+			t.Run("when mapping is found a unit should be specified on the field config", func(t *testing.T) {
-			t.Run("and cloud monitoring unit does not have a corresponding grafana unit", func(t *testing.T) {
+				data, err := loadTestFile("./test-data/1-series-response-agg-one-metric.json")
-				executors := []cloudMonitoringQueryExecutor{
+				require.NoError(t, err)
-					&cloudMonitoringTimeSeriesFilter{Params: url.Values{}, ProjectName: "test-proj", Selector: "select_slo_compliance", Service: "test-service",
+				assert.Equal(t, 1, len(data.TimeSeries))
-						Slo: "test-slo", Unit: "megaseconds"},
+
-				}
+				res := &plugins.DataQueryResult{Meta: simplejson.New(), RefID: "A"}
-				unit := executor.resolvePanelUnitFromQueries(executors)
+				query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}}
-				assert.Equal(t, "", unit)
+				err = query.parseResponse(res, data, "")
+				require.NoError(t, err)
+				frames, err := res.Dataframes.Decoded()
+				require.NoError(t, err)
+				assert.Equal(t, "Bps", frames[0].Fields[1].Config.Unit)
 			})
 
-			t.Run("and cloud monitoring unit has a corresponding grafana unit", func(t *testing.T) {
+			t.Run("when mapping is found a unit should be specified on the field config", func(t *testing.T) {
-				for key, element := range cloudMonitoringUnitMappings {
+				data, err := loadTestFile("./test-data/2-series-response-no-agg.json")
-					queries := []cloudMonitoringQueryExecutor{
+				require.NoError(t, err)
-						&cloudMonitoringTimeSeriesFilter{Params: url.Values{}, ProjectName: "test-proj", Selector: "select_slo_compliance", Service: "test-service",
+				assert.Equal(t, 3, len(data.TimeSeries))
-							Slo: "test-slo", Unit: key},
-					}
-					unit := executor.resolvePanelUnitFromQueries(queries)
-					assert.Equal(t, element, unit)
-				}
-			})
-		})
 
-		t.Run("when there are more than one query", func(t *testing.T) {
+				res := &plugins.DataQueryResult{Meta: simplejson.New(), RefID: "A"}
-			t.Run("and all target units are the same", func(t *testing.T) {
+				query := &cloudMonitoringTimeSeriesFilter{Params: url.Values{}}
-				for key, element := range cloudMonitoringUnitMappings {
+				err = query.parseResponse(res, data, "")
-					queries := []cloudMonitoringQueryExecutor{
+				require.NoError(t, err)
-						&cloudMonitoringTimeSeriesFilter{
+				frames, err := res.Dataframes.Decoded()
-							Params: url.Values{}, ProjectName: "test-proj", Selector: "select_slo_compliance",
+				require.NoError(t, err)
-							Service: "test-service1", Slo: "test-slo", Unit: key,
+				assert.Equal(t, "", frames[0].Fields[1].Config.Unit)
-						},
-						&cloudMonitoringTimeSeriesFilter{
-							Params: url.Values{}, ProjectName: "test-proj", Selector: "select_slo_compliance",
-							Service: "test-service2", Slo: "test-slo", Unit: key,
-						},
-					}
-					unit := executor.resolvePanelUnitFromQueries(queries)
-					assert.Equal(t, element, unit)
-				}
-			})
-
-			t.Run("and all target units are the same but does not have grafana mappings", func(t *testing.T) {
-				queries := []cloudMonitoringQueryExecutor{
-					&cloudMonitoringTimeSeriesFilter{
-						Params: url.Values{}, ProjectName: "test-proj", Selector: "select_slo_compliance",
-						Service: "test-service1", Slo: "test-slo", Unit: "megaseconds",
-					},
-					&cloudMonitoringTimeSeriesFilter{
-						Params: url.Values{}, ProjectName: "test-proj", Selector: "select_slo_compliance",
-						Service: "test-service2", Slo: "test-slo", Unit: "megaseconds",
-					},
-				}
-				unit := executor.resolvePanelUnitFromQueries(queries)
-				assert.Equal(t, "", unit)
-			})
-
-			t.Run("and all target units are not the same", func(t *testing.T) {
-				queries := []cloudMonitoringQueryExecutor{
-					&cloudMonitoringTimeSeriesFilter{
-						Params: url.Values{}, ProjectName: "test-proj", Selector: "select_slo_compliance",
-						Service: "test-service1", Slo: "test-slo", Unit: "bit",
-					},
-					&cloudMonitoringTimeSeriesFilter{
-						Params: url.Values{}, ProjectName: "test-proj", Selector: "select_slo_compliance",
-						Service: "test-service2", Slo: "test-slo", Unit: "min",
-					},
-				}
-				unit := executor.resolvePanelUnitFromQueries(queries)
-				assert.Equal(t, "", unit)
 			})
 		})
 

pkg/tsdb/cloudmonitoring/test-data/1-series-response-agg-one-metric.json

@@ -42,5 +42,6 @@
         }
       ]
     }
-  ]
+  ],
+  "unit": "By/s"
 }

pkg/tsdb/cloudmonitoring/test-data/2-series-response-no-agg.json

@@ -141,5 +141,6 @@
         }
       ]
     }
-  ]
+  ],
+  "unit": "10^2.%"
 }

pkg/tsdb/cloudmonitoring/time_series_filter.go

@@ -224,7 +224,7 @@ func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) parseResponse(queryRes
 	}
 	if len(response.TimeSeries) > 0 {
 		dl := timeSeriesFilter.buildDeepLink()
-		frames = addConfigData(frames, dl)
+		frames = addConfigData(frames, dl, response.Unit)
 	}
 
 	queryRes.Dataframes = plugins.NewDecodedDataFrames(frames)
@@ -392,7 +392,3 @@ func setDisplayNameAsFieldName(f *data.Field) {
 func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) getRefID() string {
 	return timeSeriesFilter.RefID
 }
-
-func (timeSeriesFilter *cloudMonitoringTimeSeriesFilter) getUnit() string {
-	return timeSeriesFilter.Unit
-}

pkg/tsdb/cloudmonitoring/time_series_query.go

@@ -247,7 +247,7 @@ func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) parseResponse(queryRes *pl
 	}
 	if len(response.TimeSeriesData) > 0 {
 		dl := timeSeriesQuery.buildDeepLink()
-		frames = addConfigData(frames, dl)
+		frames = addConfigData(frames, dl, response.Unit)
 	}
 
 	queryRes.Dataframes = plugins.NewDecodedDataFrames(frames)
@@ -375,7 +375,3 @@ func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) buildDeepLink() string {
 func (timeSeriesQuery cloudMonitoringTimeSeriesQuery) getRefID() string {
 	return timeSeriesQuery.RefID
 }
-
-func (timeSeriesQuery *cloudMonitoringTimeSeriesQuery) getUnit() string {
-	return timeSeriesQuery.Unit
-}

pkg/tsdb/cloudmonitoring/types.go

@@ -16,7 +16,6 @@ type (
 		parseToAnnotations(queryRes *plugins.DataQueryResult, data cloudMonitoringResponse, title string, text string, tags string) error
 		buildDeepLink() string
 		getRefID() string
-		getUnit() string
 	}
 
 	// Used to build time series filters
@@ -30,7 +29,6 @@ type (
 		Selector    string
 		Service     string
 		Slo         string
-		Unit        string
 	}
 
 	// Used to build MQL queries
@@ -41,7 +39,6 @@ type (
 		IntervalMS  int64
 		AliasBy     string
 		timeRange   plugins.DataTimeRange
-		Unit        string
 	}
 
 	metricQuery struct {
@@ -56,7 +53,6 @@ type (
 		View               string
 		EditorMode         string
 		Query              string
-		Unit               string
 	}
 
 	sloQuery struct {
@@ -97,6 +93,7 @@ type (
 		TimeSeries           []timeSeries         `json:"timeSeries"`
 		TimeSeriesDescriptor timeSeriesDescriptor `json:"timeSeriesDescriptor"`
 		TimeSeriesData       timeSeriesData       `json:"timeSeriesData"`
+		Unit                 string               `json:"unit"`
 	}
 )