CloudWatch: Improve metric label parsing (#84835)

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -60,6 +60,7 @@ Some features are enabled by default. You can disable these feature by setting t
 | `enablePluginsTracingByDefault`          | Enable plugin tracing for all external plugins                                                                                                                                                                               | Yes                |
 | `alertingQueryOptimization`              | Optimizes eligible queries in order to reduce load on datasources                                                                                                                                                            |                    |
 | `betterPageScrolling`                    | Removes CustomScrollbar from the UI, relying on native browser scrollbars                                                                                                                                                    | Yes                |
+| `cloudWatchNewLabelParsing`              | Updates CloudWatch label parsing to be more accurate                                                                                                                                                                         | Yes                |
 
 ## Preview feature toggles
 

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -179,4 +179,5 @@ export interface FeatureToggles {
   usePrometheusFrontendPackage?: boolean;
   oauthRequireSubClaim?: boolean;
   newDashboardWithFiltersAndGroupBy?: boolean;
+  cloudWatchNewLabelParsing?: boolean;
 }

pkg/services/featuremgmt/registry.go

@@ -1205,6 +1205,15 @@ var (
 			HideFromDocs:      true,
 			HideFromAdminPage: true,
 		},
+		{
+			Name:           "cloudWatchNewLabelParsing",
+			Description:    "Updates CloudWatch label parsing to be more accurate",
+			Stage:          FeatureStageGeneralAvailability,
+			Expression:     "true", // enabled by default
+			Owner:          awsDatasourcesSquad,
+			FrontendOnly:   false,
+			AllowSelfServe: false,
+		},
 	}
 )
 

pkg/services/featuremgmt/toggles_gen.csv

@@ -160,3 +160,4 @@ ssoSettingsSAML,experimental,@grafana/identity-access-team,false,false,false
 usePrometheusFrontendPackage,experimental,@grafana/observability-metrics,false,false,true
 oauthRequireSubClaim,experimental,@grafana/identity-access-team,false,false,false
 newDashboardWithFiltersAndGroupBy,experimental,@grafana/dashboards-squad,false,false,false
+cloudWatchNewLabelParsing,GA,@grafana/aws-datasources,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -650,4 +650,8 @@ const (
 	// FlagNewDashboardWithFiltersAndGroupBy
 	// Enables filters and group by variables on all new dashboards. Variables are added only if default data source supports filtering.
 	FlagNewDashboardWithFiltersAndGroupBy = "newDashboardWithFiltersAndGroupBy"
+
+	// FlagCloudWatchNewLabelParsing
+	// Updates CloudWatch label parsing to be more accurate
+	FlagCloudWatchNewLabelParsing = "cloudWatchNewLabelParsing"
 )

pkg/tsdb/cloudwatch/features/features.go

@@ -9,6 +9,7 @@ import (
 const (
 	FlagCloudWatchCrossAccountQuerying = "cloudWatchCrossAccountQuerying"
 	FlagCloudWatchBatchQueries         = "cloudWatchBatchQueries"
+	FlagCloudWatchNewLabelParsing      = "cloudWatchNewLabelParsing"
 )
 
 func IsEnabled(ctx context.Context, feature string) bool {

pkg/tsdb/cloudwatch/metric_data_input_builder.go

@@ -1,6 +1,7 @@
 package cloudwatch
 
 import (
+	"context"
 	"time"
 
 	"github.com/aws/aws-sdk-go/aws"
@@ -9,7 +10,7 @@ import (
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
 )
 
-func (e *cloudWatchExecutor) buildMetricDataInput(startTime time.Time, endTime time.Time,
+func (e *cloudWatchExecutor) buildMetricDataInput(ctx context.Context, startTime time.Time, endTime time.Time,
 	queries []*models.CloudWatchQuery) (*cloudwatch.GetMetricDataInput, error) {
 	metricDataInput := &cloudwatch.GetMetricDataInput{
 		StartTime: aws.Time(startTime),
@@ -26,7 +27,7 @@ func (e *cloudWatchExecutor) buildMetricDataInput(startTime time.Time, endTime t
 	}
 
 	for _, query := range queries {
-		metricDataQuery, err := e.buildMetricDataQuery(query)
+		metricDataQuery, err := e.buildMetricDataQuery(ctx, query)
 		if err != nil {
 			return nil, &models.QueryError{Err: err, RefID: query.RefId}
 		}

pkg/tsdb/cloudwatch/metric_data_input_builder_test.go

@@ -1,6 +1,7 @@
 package cloudwatch
 
 import (
+	"context"
 	"testing"
 	"time"
 
@@ -33,7 +34,7 @@ func TestMetricDataInputBuilder(t *testing.T) {
 
 			from := now.Add(time.Hour * -2)
 			to := now.Add(time.Hour * -1)
-			mdi, err := executor.buildMetricDataInput(from, to, []*models.CloudWatchQuery{query})
+			mdi, err := executor.buildMetricDataInput(context.Background(), from, to, []*models.CloudWatchQuery{query})
 
 			assert.NoError(t, err)
 			require.NotNil(t, mdi)

pkg/tsdb/cloudwatch/metric_data_query_builder.go

@@ -1,6 +1,7 @@
 package cloudwatch
 
 import (
+	"context"
 	"fmt"
 	"sort"
 	"strconv"
@@ -9,10 +10,13 @@ import (
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
 
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/features"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
 )
 
-func (e *cloudWatchExecutor) buildMetricDataQuery(query *models.CloudWatchQuery) (*cloudwatch.MetricDataQuery, error) {
+const keySeparator = "|&|"
+
+func (e *cloudWatchExecutor) buildMetricDataQuery(ctx context.Context, query *models.CloudWatchQuery) (*cloudwatch.MetricDataQuery, error) {
 	mdq := &cloudwatch.MetricDataQuery{
 		Id:         aws.String(query.Id),
 		ReturnData: aws.Bool(query.ReturnData),
@@ -31,6 +35,9 @@ func (e *cloudWatchExecutor) buildMetricDataQuery(query *models.CloudWatchQuery)
 		mdq.Expression = aws.String(query.SqlExpression)
 	case models.GMDApiModeInferredSearchExpression:
 		mdq.Expression = aws.String(buildSearchExpression(query, query.Statistic))
+		if features.IsEnabled(ctx, features.FlagCloudWatchNewLabelParsing) {
+			mdq.Label = aws.String(buildSearchExpressionLabel(query))
+		}
 	case models.GMDApiModeMetricStat:
 		mdq.MetricStat = &cloudwatch.MetricStat{
 			Metric: &cloudwatch.Metric{
@@ -60,6 +67,10 @@ func (e *cloudWatchExecutor) buildMetricDataQuery(query *models.CloudWatchQuery)
 	return mdq, nil
 }
 
+func isSingleValue(values []string) bool {
+	return len(values) == 1 && values[0] != "*"
+}
+
 func buildSearchExpression(query *models.CloudWatchQuery, stat string) string {
 	knownDimensions := make(map[string][]string)
 	dimensionNames := []string{}
@@ -120,6 +131,25 @@ func buildSearchExpression(query *models.CloudWatchQuery, stat string) string {
 	return fmt.Sprintf(`REMOVE_EMPTY(SEARCH('%s', '%s', %s))`, namespaceSearchTermAndAccount, stat, strconv.Itoa(query.Period))
 }
 
+func buildSearchExpressionLabel(query *models.CloudWatchQuery) string {
+	label := "${LABEL}"
+	if len(query.Label) > 0 {
+		label = query.Label
+	}
+
+	multiDims := []string{}
+	for key, values := range query.Dimensions {
+		if !isSingleValue(values) {
+			multiDims = append(multiDims, key)
+		}
+	}
+	sort.Strings(multiDims)
+	for _, key := range multiDims {
+		label += fmt.Sprintf("%s${PROP('Dim.%s')}", keySeparator, key)
+	}
+	return label
+}
+
 func escapeDoubleQuotes(arr []string) []string {
 	result := []string{}
 	for _, value := range arr {

pkg/tsdb/cloudwatch/metric_data_query_builder_test.go

@@ -1,23 +1,25 @@
 package cloudwatch
 
 import (
+	"context"
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/log"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/features"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestMetricDataQueryBuilder(t *testing.T) {
+	executor := newExecutor(nil, log.NewNullLogger())
 	t.Run("buildMetricDataQuery", func(t *testing.T) {
 		t.Run("should use metric stat", func(t *testing.T) {
-			executor := newExecutor(nil, log.NewNullLogger())
 			query := getBaseQuery()
 			query.MetricEditorMode = models.MetricEditorModeBuilder
 			query.MetricQueryType = models.MetricQueryTypeSearch
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 			require.NoError(t, err)
 			require.Empty(t, mdq.Expression)
 			assert.Equal(t, query.MetricName, *mdq.MetricStat.Metric.MetricName)
@@ -25,57 +27,52 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 		})
 
 		t.Run("should pass AccountId in metric stat query", func(t *testing.T) {
-			executor := newExecutor(nil, log.NewNullLogger())
 			query := getBaseQuery()
 			query.MetricEditorMode = models.MetricEditorModeBuilder
 			query.MetricQueryType = models.MetricQueryTypeSearch
 			query.AccountId = aws.String("some account id")
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 			require.NoError(t, err)
 			assert.Equal(t, "some account id", *mdq.AccountId)
 		})
 
 		t.Run("should leave AccountId in metric stat query", func(t *testing.T) {
-			executor := newExecutor(nil, log.NewNullLogger())
 			query := getBaseQuery()
 			query.MetricEditorMode = models.MetricEditorModeBuilder
 			query.MetricQueryType = models.MetricQueryTypeSearch
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 			require.NoError(t, err)
 			assert.Nil(t, mdq.AccountId)
 		})
 
 		t.Run("should use custom built expression", func(t *testing.T) {
-			executor := newExecutor(nil, log.NewNullLogger())
 			query := getBaseQuery()
 			query.MetricEditorMode = models.MetricEditorModeBuilder
 			query.MetricQueryType = models.MetricQueryTypeSearch
 			query.MatchExact = false
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 			require.NoError(t, err)
 			require.Nil(t, mdq.MetricStat)
 			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"="lb1"', '', 300))`, *mdq.Expression)
 		})
 
 		t.Run("should use sql expression", func(t *testing.T) {
-			executor := newExecutor(nil, log.NewNullLogger())
 			query := getBaseQuery()
 			query.MetricEditorMode = models.MetricEditorModeRaw
 			query.MetricQueryType = models.MetricQueryTypeQuery
 			query.SqlExpression = `SELECT SUM(CPUUTilization) FROM "AWS/EC2"`
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 			require.NoError(t, err)
 			require.Nil(t, mdq.MetricStat)
 			assert.Equal(t, query.SqlExpression, *mdq.Expression)
 		})
 
 		t.Run("should use user defined math expression", func(t *testing.T) {
-			executor := newExecutor(nil, log.NewNullLogger())
 			query := getBaseQuery()
 			query.MetricEditorMode = models.MetricEditorModeRaw
 			query.MetricQueryType = models.MetricQueryTypeSearch
 			query.Expression = `SUM(x+y)`
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 			require.NoError(t, err)
 			require.Nil(t, mdq.MetricStat)
 			assert.Equal(t, query.Expression, *mdq.Expression)
@@ -88,7 +85,7 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 			query.MetricQueryType = models.MetricQueryTypeSearch
 			query.MatchExact = false
 			query.Expression = `SUM([a,b])`
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 			require.NoError(t, err)
 			require.Nil(t, mdq.MetricStat)
 			assert.Equal(t, int64(300), *mdq.Period)
@@ -100,7 +97,7 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 			query := getBaseQuery()
 			query.Label = "some label"
 
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 
 			assert.NoError(t, err)
 			require.NotNil(t, mdq.Label)
@@ -112,7 +109,7 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 			query := getBaseQuery()
 			query.Label = ""
 
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 
 			assert.NoError(t, err)
 			assert.Nil(t, mdq.Label)
@@ -129,7 +126,7 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				AccountId:  aws.String("all"),
 			}
 
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 
 			assert.NoError(t, err)
 			require.Nil(t, mdq.MetricStat)
@@ -147,7 +144,7 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				AccountId:  aws.String("12345"),
 			}
 
-			mdq, err := executor.buildMetricDataQuery(query)
+			mdq, err := executor.buildMetricDataQuery(context.Background(), query)
 
 			assert.NoError(t, err)
 			require.Nil(t, mdq.MetricStat)
@@ -168,10 +165,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","LoadBalancer"} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","LoadBalancer"} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
 		})
 
 		t.Run("Query has three dimension values for two given dimension keys", func(t *testing.T) {
@@ -185,26 +187,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId","LoadBalancer"} MetricName="CPUUtilization" "InstanceId"=("i-123" OR "i-456" OR "i-789") "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, res)
+			require.NoError(t, err)
-		})
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId","LoadBalancer"} MetricName="CPUUtilization" "InstanceId"=("i-123" OR "i-456" OR "i-789") "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, *mdq.Expression)
-
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.InstanceId')}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
-		t.Run("No OR operator was added if a star was used for dimension value", func(t *testing.T) {
-			query := &models.CloudWatchQuery{
-				Namespace:  "AWS/EC2",
-				MetricName: "CPUUtilization",
-				Dimensions: map[string][]string{
-					"LoadBalancer": {"*"},
-				},
-				Period:     300,
-				Expression: "",
-				MatchExact: matchExact,
-			}
-
-			res := buildSearchExpression(query, "Average")
-			assert.NotContains(t, res, "OR")
 		})
 
 		t.Run("Query has one dimension key with a * value", func(t *testing.T) {
@@ -217,10 +208,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","LoadBalancer"} MetricName="CPUUtilization"', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","LoadBalancer"} MetricName="CPUUtilization"', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
 		})
 
 		t.Run("Query has three dimension values for two given dimension keys, and one value is a star", func(t *testing.T) {
@@ -234,10 +230,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId","LoadBalancer"} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId","LoadBalancer"} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.InstanceId')}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
 		})
 
 		t.Run("Query has multiple dimensions and an account Id", func(t *testing.T) {
@@ -252,10 +253,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Expression:       "",
 				MatchExact:       matchExact,
 				AccountId:        aws.String("some account id"),
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId","LoadBalancer"} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") :aws.AccountId="some account id"', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/EC2","InstanceId","LoadBalancer"} MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") :aws.AccountId="some account id"', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.InstanceId')}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
 		})
 
 		t.Run("Query has a dimension key with a space", func(t *testing.T) {
@@ -263,15 +269,20 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Namespace:  "AWS/Kafka",
 				MetricName: "CpuUser",
 				Dimensions: map[string][]string{
-					"Cluster Name": {"dev-cluster"},
+					"Cluster Name": {"dev-cluster", "prod-cluster"},
 				},
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/Kafka","Cluster Name"} MetricName="CpuUser" "Cluster Name"="dev-cluster"', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"AWS/Kafka","Cluster Name"} MetricName="CpuUser" "Cluster Name"=("dev-cluster" OR "prod-cluster")', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.Cluster Name')}", *mdq.Label)
 		})
 
 		t.Run("Query has a custom namespace contains spaces", func(t *testing.T) {
@@ -285,10 +296,38 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"Test-API Cache by Minute","InstanceId","LoadBalancer"} MetricName="CpuUser" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"Test-API Cache by Minute","InstanceId","LoadBalancer"} MetricName="CpuUser" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.InstanceId')}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
+		})
+
+		t.Run("Query has a custom label", func(t *testing.T) {
+			query := &models.CloudWatchQuery{
+				Namespace:  "CPUUtilization",
+				MetricName: "CpuUser",
+				Dimensions: map[string][]string{
+					"LoadBalancer": {"lb1"},
+					"InstanceId":   {"i-123", "*", "i-789"},
+				},
+				Period:           300,
+				Expression:       "",
+				MatchExact:       matchExact,
+				Statistic:        "Average",
+				Label:            "LB: ${PROP('Dim.LoadBalancer')",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
+			}
+
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('{"CPUUtilization","InstanceId","LoadBalancer"} MetricName="CpuUser" "LoadBalancer"="lb1"', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "LB: ${PROP('Dim.LoadBalancer')|&|${PROP('Dim.InstanceId')}", *mdq.Label)
 		})
 	})
 
@@ -305,10 +344,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
 		})
 
 		t.Run("Query has three dimension values for two given dimension keys", func(t *testing.T) {
@@ -322,10 +366,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "InstanceId"=("i-123" OR "i-456" OR "i-789") "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "InstanceId"=("i-123" OR "i-456" OR "i-789") "LoadBalancer"=("lb1" OR "lb2" OR "lb3")', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.InstanceId')}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
 		})
 
 		t.Run("Query has one dimension key with a * value", func(t *testing.T) {
@@ -338,10 +387,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
 		})
 
 		t.Run("query has three dimension values for two given dimension keys, and one value is a star", func(t *testing.T) {
@@ -355,10 +409,15 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Period:           300,
 				Expression:       "",
 				MatchExact:       matchExact,
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") "InstanceId"', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") "InstanceId"', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.InstanceId')}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
 		})
 
 		t.Run("query has multiple dimensions and an account Id", func(t *testing.T) {
@@ -373,10 +432,38 @@ func TestMetricDataQueryBuilder(t *testing.T) {
 				Expression:       "",
 				MatchExact:       matchExact,
 				AccountId:        aws.String("some account id"),
+				Statistic:        "Average",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
 			}
 
-			res := buildSearchExpression(query, "Average")
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
-			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") "InstanceId" :aws.AccountId="some account id"', 'Average', 300))`, res)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"=("lb1" OR "lb2" OR "lb3") "InstanceId" :aws.AccountId="some account id"', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "${LABEL}|&|${PROP('Dim.InstanceId')}|&|${PROP('Dim.LoadBalancer')}", *mdq.Label)
+		})
+
+		t.Run("Query has a custom label", func(t *testing.T) {
+			query := &models.CloudWatchQuery{
+				Namespace:  "AWS/EC2",
+				MetricName: "CPUUtilization",
+				Dimensions: map[string][]string{
+					"LoadBalancer": {"lb1"},
+					"InstanceId":   {"i-123", "*", "i-789"},
+				},
+				Period:           300,
+				Expression:       "",
+				MatchExact:       matchExact,
+				Statistic:        "Average",
+				Label:            "LB: ${PROP('Dim.LoadBalancer')",
+				MetricQueryType:  models.MetricQueryTypeSearch,
+				MetricEditorMode: models.MetricEditorModeBuilder,
+			}
+
+			mdq, err := executor.buildMetricDataQuery(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), query)
+			require.NoError(t, err)
+			assert.Equal(t, `REMOVE_EMPTY(SEARCH('Namespace="AWS/EC2" MetricName="CPUUtilization" "LoadBalancer"="lb1" "InstanceId"', 'Average', 300))`, *mdq.Expression)
+			assert.Equal(t, "LB: ${PROP('Dim.LoadBalancer')|&|${PROP('Dim.InstanceId')}", *mdq.Label)
 		})
 	})
 

pkg/tsdb/cloudwatch/response_parser.go

@@ -1,6 +1,7 @@
 package cloudwatch
 
 import (
+	"context"
 	"fmt"
 	"regexp"
 	"sort"
@@ -10,13 +11,14 @@ import (
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/features"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
 )
 
 // matches a dynamic label
 var dynamicLabel = regexp.MustCompile(`\$\{.+\}`)
 
-func (e *cloudWatchExecutor) parseResponse(startTime time.Time, endTime time.Time, metricDataOutputs []*cloudwatch.GetMetricDataOutput,
+func (e *cloudWatchExecutor) parseResponse(ctx context.Context, startTime time.Time, endTime time.Time, metricDataOutputs []*cloudwatch.GetMetricDataOutput,
 	queries []*models.CloudWatchQuery) ([]*responseWrapper, error) {
 	aggregatedResponse := aggregateResponse(metricDataOutputs)
 	queriesById := map[string]*models.CloudWatchQuery{}
@@ -34,7 +36,7 @@ func (e *cloudWatchExecutor) parseResponse(startTime time.Time, endTime time.Tim
 		}
 
 		var err error
-		dataRes.Frames, err = buildDataFrames(startTime, endTime, response, queryRow)
+		dataRes.Frames, err = buildDataFrames(ctx, startTime, endTime, response, queryRow)
 		if err != nil {
 			return nil, err
 		}
@@ -87,6 +89,31 @@ func aggregateResponse(getMetricDataOutputs []*cloudwatch.GetMetricDataOutput) m
 	return responseByID
 }
 
+func parseLabels(cloudwatchLabel string, query *models.CloudWatchQuery) (string, data.Labels) {
+	dims := make([]string, 0, len(query.Dimensions))
+	for k := range query.Dimensions {
+		dims = append(dims, k)
+	}
+	sort.Strings(dims)
+
+	splitLabels := strings.Split(cloudwatchLabel, keySeparator)
+	// The first part is the name of the time series, followed by the labels
+	labelsIndex := 1
+
+	labels := data.Labels{}
+	for _, dim := range dims {
+		values := query.Dimensions[dim]
+		if isSingleValue(values) {
+			labels[dim] = values[0]
+			continue
+		}
+
+		labels[dim] = splitLabels[labelsIndex]
+		labelsIndex++
+	}
+	return splitLabels[0], labels
+}
+
 func getLabels(cloudwatchLabel string, query *models.CloudWatchQuery) data.Labels {
 	dims := make([]string, 0, len(query.Dimensions))
 	for k := range query.Dimensions {
@@ -111,7 +138,7 @@ func getLabels(cloudwatchLabel string, query *models.CloudWatchQuery) data.Label
 	return labels
 }
 
-func buildDataFrames(startTime time.Time, endTime time.Time, aggregatedResponse models.QueryRowResponse,
+func buildDataFrames(ctx context.Context, startTime time.Time, endTime time.Time, aggregatedResponse models.QueryRowResponse,
 	query *models.CloudWatchQuery) (data.Frames, error) {
 	frames := data.Frames{}
 	hasStaticLabel := query.Label != "" && !dynamicLabel.MatchString(query.Label)
@@ -127,6 +154,9 @@ func buildDataFrames(startTime time.Time, endTime time.Time, aggregatedResponse
 		// In case a multi-valued dimension is used and the cloudwatch query yields no values, create one empty time
 		// series for each dimension value. Use that dimension value to expand the alias field
 		if len(metric.Values) == 0 && query.IsMultiValuedDimensionExpression() {
+			if features.IsEnabled(ctx, features.FlagCloudWatchNewLabelParsing) {
+				label, _, _ = strings.Cut(label, keySeparator)
+			}
 			series := 0
 			multiValuedDimension := ""
 			for key, values := range query.Dimensions {
@@ -163,7 +193,13 @@ func buildDataFrames(startTime time.Time, endTime time.Time, aggregatedResponse
 			continue
 		}
 
-		labels := getLabels(label, query)
+		name := label
+		var labels data.Labels
+		if features.IsEnabled(ctx, features.FlagCloudWatchNewLabelParsing) {
+			name, labels = parseLabels(label, query)
+		} else {
+			labels = getLabels(label, query)
+		}
 		timestamps := []*time.Time{}
 		points := []*float64{}
 		for j, t := range metric.Timestamps {
@@ -175,7 +211,6 @@ func buildDataFrames(startTime time.Time, endTime time.Time, aggregatedResponse
 		timeField := data.NewField(data.TimeSeriesTimeFieldName, nil, timestamps)
 		valueField := data.NewField(data.TimeSeriesValueFieldName, labels, points)
 
-		name := label
 		// CloudWatch appends the dimensions to the returned label if the query label is not dynamic, so static labels need to be set
 		if hasStaticLabel {
 			name = query.Label

pkg/tsdb/cloudwatch/response_parser_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/features"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -136,16 +137,17 @@ func TestCloudWatchResponseParser(t *testing.T) {
 	})
 }
 
-func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
+func Test_buildDataFrames_parse_label_to_name_and_labels(t *testing.T) {
 	startTime := time.Now()
 	endTime := startTime.Add(2 * time.Hour)
-	t.Run("using exact match", func(t *testing.T) {
+
+	t.Run("using multi filter", func(t *testing.T) {
 		timestamp := time.Unix(0, 0)
 		response := &models.QueryRowResponse{
 			Metrics: []*cloudwatch.MetricDataResult{
 				{
 					Id:    aws.String("id1"),
-					Label: aws.String("lb1"),
+					Label: aws.String("lb1|&|lb1"),
 					Timestamps: []*time.Time{
 						aws.Time(timestamp),
 						aws.Time(timestamp.Add(time.Minute)),
@@ -160,7 +162,7 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 				},
 				{
 					Id:    aws.String("id2"),
-					Label: aws.String("lb2"),
+					Label: aws.String("lb2|&|lb2"),
 					Timestamps: []*time.Time{
 						aws.Time(timestamp),
 						aws.Time(timestamp.Add(time.Minute)),
@@ -189,26 +191,29 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			Period:           60,
 			MetricQueryType:  models.MetricQueryTypeSearch,
 			MetricEditorMode: models.MetricEditorModeBuilder,
+			MatchExact:       true,
 		}
-		frames, err := buildDataFrames(startTime, endTime, *response, query)
+		frames, err := buildDataFrames(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), startTime, endTime, *response, query)
 		require.NoError(t, err)
 
 		frame1 := frames[0]
 		assert.Equal(t, "lb1", frame1.Name)
 		assert.Equal(t, "lb1", frame1.Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "tg", frame1.Fields[1].Labels["TargetGroup"])
 
 		frame2 := frames[1]
 		assert.Equal(t, "lb2", frame2.Name)
 		assert.Equal(t, "lb2", frame2.Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "tg", frame2.Fields[1].Labels["TargetGroup"])
 	})
 
-	t.Run("using wildcard", func(t *testing.T) {
+	t.Run("using multiple wildcard filters", func(t *testing.T) {
 		timestamp := time.Unix(0, 0)
 		response := &models.QueryRowResponse{
 			Metrics: []*cloudwatch.MetricDataResult{
 				{
 					Id:    aws.String("lb3"),
-					Label: aws.String("some label lb3"),
+					Label: aws.String("some label lb3|&|inst1|&|balancer 1"),
 					Timestamps: []*time.Time{
 						aws.Time(timestamp),
 						aws.Time(timestamp.Add(time.Minute)),
@@ -223,7 +228,7 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 				},
 				{
 					Id:    aws.String("lb4"),
-					Label: aws.String("some label lb4"),
+					Label: aws.String("some label lb4|&|inst2|&|balancer 2"),
 					Timestamps: []*time.Time{
 						aws.Time(timestamp),
 						aws.Time(timestamp.Add(time.Minute)),
@@ -246,6 +251,7 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			MetricName: "TargetResponseTime",
 			Dimensions: map[string][]string{
 				"LoadBalancer": {"*"},
+				"InstanceType": {"*"},
 				"TargetGroup":  {"tg"},
 			},
 			Statistic:        "Average",
@@ -253,20 +259,28 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			MetricQueryType:  models.MetricQueryTypeSearch,
 			MetricEditorMode: models.MetricEditorModeBuilder,
 		}
-		frames, err := buildDataFrames(startTime, endTime, *response, query)
+		frames, err := buildDataFrames(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), startTime, endTime, *response, query)
 		require.NoError(t, err)
 
 		assert.Equal(t, "some label lb3", frames[0].Name)
+		assert.Equal(t, "balancer 1", frames[0].Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "inst1", frames[0].Fields[1].Labels["InstanceType"])
+		assert.Equal(t, "tg", frames[0].Fields[1].Labels["TargetGroup"])
+
 		assert.Equal(t, "some label lb4", frames[1].Name)
+		assert.Equal(t, "balancer 2", frames[1].Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "inst2", frames[1].Fields[1].Labels["InstanceType"])
+		assert.Equal(t, "tg", frames[1].Fields[1].Labels["TargetGroup"])
 	})
 
 	t.Run("when no values are returned and a multi-valued template variable is used", func(t *testing.T) {
 		timestamp := time.Unix(0, 0)
+		// When there are no results, CloudWatch sets the label values to --
 		response := &models.QueryRowResponse{
 			Metrics: []*cloudwatch.MetricDataResult{
 				{
 					Id:    aws.String("lb3"),
-					Label: aws.String("some label"),
+					Label: aws.String("some label|&|--"),
 					Timestamps: []*time.Time{
 						aws.Time(timestamp),
 						aws.Time(timestamp.Add(time.Minute)),
@@ -290,21 +304,24 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			MetricQueryType:  models.MetricQueryTypeSearch,
 			MetricEditorMode: models.MetricEditorModeBuilder,
 		}
-		frames, err := buildDataFrames(startTime, endTime, *response, query)
+		frames, err := buildDataFrames(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), startTime, endTime, *response, query)
 		require.NoError(t, err)
 
 		assert.Len(t, frames, 2)
 		assert.Equal(t, "some label", frames[0].Name)
+		assert.Equal(t, "lb1", frames[0].Fields[1].Labels["LoadBalancer"])
 		assert.Equal(t, "some label", frames[1].Name)
+		assert.Equal(t, "lb2", frames[1].Fields[1].Labels["LoadBalancer"])
 	})
 
 	t.Run("when no values are returned and a multi-valued template variable and two single-valued dimensions are used", func(t *testing.T) {
 		timestamp := time.Unix(0, 0)
+		// When there are no results, CloudWatch sets the label values to --
 		response := &models.QueryRowResponse{
 			Metrics: []*cloudwatch.MetricDataResult{
 				{
 					Id:    aws.String("lb3"),
-					Label: aws.String("some label"),
+					Label: aws.String("some label|&|--"),
 					Timestamps: []*time.Time{
 						aws.Time(timestamp),
 						aws.Time(timestamp.Add(time.Minute)),
@@ -331,15 +348,22 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			MetricQueryType:  models.MetricQueryTypeSearch,
 			MetricEditorMode: models.MetricEditorModeBuilder,
 		}
-		frames, err := buildDataFrames(startTime, endTime, *response, query)
+		frames, err := buildDataFrames(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), startTime, endTime, *response, query)
 		require.NoError(t, err)
 
 		assert.Len(t, frames, 2)
 		assert.Equal(t, "some label", frames[0].Name)
+		assert.Equal(t, "lb1", frames[0].Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "micro", frames[0].Fields[1].Labels["InstanceType"])
+		assert.Equal(t, "res", frames[0].Fields[1].Labels["Resource"])
+
 		assert.Equal(t, "some label", frames[1].Name)
+		assert.Equal(t, "lb2", frames[1].Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "micro", frames[1].Fields[1].Labels["InstanceType"])
+		assert.Equal(t, "res", frames[1].Fields[1].Labels["Resource"])
 	})
 
-	t.Run("when using SQL queries", func(t *testing.T) {
+	t.Run("when not using multi-value dimension filters", func(t *testing.T) {
 		timestamp := time.Unix(0, 0)
 		response := &models.QueryRowResponse{
 			Metrics: []*cloudwatch.MetricDataResult{
@@ -370,10 +394,13 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			MetricQueryType:  models.MetricQueryTypeQuery,
 			MetricEditorMode: models.MetricEditorModeRaw,
 		}
-		frames, err := buildDataFrames(startTime, endTime, *response, query)
+		frames, err := buildDataFrames(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), startTime, endTime, *response, query)
 		require.NoError(t, err)
 
 		assert.Equal(t, "some label", frames[0].Name)
+		assert.Equal(t, "lb1", frames[0].Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "micro", frames[0].Fields[1].Labels["InstanceType"])
+		assert.Equal(t, "res", frames[0].Fields[1].Labels["Resource"])
 	})
 
 	t.Run("when non-static label set on query", func(t *testing.T) {
@@ -382,7 +409,7 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			Metrics: []*cloudwatch.MetricDataResult{
 				{
 					Id:    aws.String("lb3"),
-					Label: aws.String("some label"),
+					Label: aws.String("some label|&|res"),
 					Timestamps: []*time.Time{
 						aws.Time(timestamp),
 					},
@@ -400,7 +427,7 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			Dimensions: map[string][]string{
 				"LoadBalancer": {"lb1"},
 				"InstanceType": {"micro"},
-				"Resource":     {"res"},
+				"Resource":     {"*"},
 			},
 			Statistic:        "Average",
 			Period:           60,
@@ -408,19 +435,22 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			MetricEditorMode: models.MetricEditorModeBuilder,
 			Label:            "set ${AVG} label",
 		}
-		frames, err := buildDataFrames(startTime, endTime, *response, query)
+		frames, err := buildDataFrames(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), startTime, endTime, *response, query)
 		require.NoError(t, err)
 
 		assert.Equal(t, "some label", frames[0].Name)
+		assert.Equal(t, "lb1", frames[0].Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "micro", frames[0].Fields[1].Labels["InstanceType"])
+		assert.Equal(t, "res", frames[0].Fields[1].Labels["Resource"])
 	})
 
-	t.Run("unless static label set on query", func(t *testing.T) {
+	t.Run("when static label set on query", func(t *testing.T) {
 		timestamp := time.Unix(0, 0)
 		response := &models.QueryRowResponse{
 			Metrics: []*cloudwatch.MetricDataResult{
 				{
 					Id:    aws.String("lb3"),
-					Label: aws.String("some label"),
+					Label: aws.String("some label|&|res"),
 					Timestamps: []*time.Time{
 						aws.Time(timestamp),
 					},
@@ -438,7 +468,7 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			Dimensions: map[string][]string{
 				"LoadBalancer": {"lb1"},
 				"InstanceType": {"micro"},
-				"Resource":     {"res"},
+				"Resource":     {"*"},
 			},
 			Statistic:        "Average",
 			Period:           60,
@@ -446,10 +476,13 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			MetricEditorMode: models.MetricEditorModeBuilder,
 			Label:            "actual",
 		}
-		frames, err := buildDataFrames(startTime, endTime, *response, query)
+		frames, err := buildDataFrames(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), startTime, endTime, *response, query)
 		require.NoError(t, err)
 
 		assert.Equal(t, "actual", frames[0].Name)
+		assert.Equal(t, "lb1", frames[0].Fields[1].Labels["LoadBalancer"])
+		assert.Equal(t, "micro", frames[0].Fields[1].Labels["InstanceType"])
+		assert.Equal(t, "res", frames[0].Fields[1].Labels["Resource"])
 	})
 
 	t.Run("Parse cloudwatch response", func(t *testing.T) {
@@ -488,7 +521,7 @@ func Test_buildDataFrames_uses_response_label_as_frame_name(t *testing.T) {
 			MetricQueryType:  models.MetricQueryTypeSearch,
 			MetricEditorMode: models.MetricEditorModeBuilder,
 		}
-		frames, err := buildDataFrames(startTime, endTime, *response, query)
+		frames, err := buildDataFrames(contextWithFeaturesEnabled(features.FlagCloudWatchNewLabelParsing), startTime, endTime, *response, query)
 		require.NoError(t, err)
 
 		frame := frames[0]

pkg/tsdb/cloudwatch/time_series_query.go

@@ -86,7 +86,7 @@ func (e *cloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, req *ba
 					return err
 				}
 
-				metricDataInput, err := e.buildMetricDataInput(startTime, endTime, requestQueries)
+				metricDataInput, err := e.buildMetricDataInput(ctx, startTime, endTime, requestQueries)
 				if err != nil {
 					return err
 				}
@@ -96,12 +96,14 @@ func (e *cloudWatchExecutor) executeTimeSeriesQuery(ctx context.Context, req *ba
 					return err
 				}
 
+				if !features.IsEnabled(ctx, features.FlagCloudWatchNewLabelParsing) {
 					requestQueries, err = e.getDimensionValuesForWildcards(ctx, region, client, requestQueries, instance.tagValueCache, instance.Settings.GrafanaSettings.ListMetricsPageLimit)
 					if err != nil {
 						return err
 					}
+				}
 
-				res, err := e.parseResponse(startTime, endTime, mdo, requestQueries)
+				res, err := e.parseResponse(ctx, startTime, endTime, mdo, requestQueries)
 				if err != nil {
 					return err
 				}