Cloudwatch: Fix Unexpected error (#74420)

Fix unexpected error when creating a new cloudwatch datasource.

Involves a fair amount of refactoring, so if this causes unexpected issues related to region fetching we can turn this off with the cloudwatchNewRegionsHandler feature toggle, although we do not predict it will so we are enabling it to default to true and hope to remove it shortly.

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -27,6 +27,7 @@ Some features are enabled by default. You can disable these feature by setting t
 | `cloudWatchCrossAccountQuerying`                 | Enables cross-account querying in CloudWatch datasources                                                                                                                                            | Yes                |
 | `redshiftAsyncQueryDataSupport`                  | Enable async query data support for Redshift                                                                                                                                                        | Yes                |
 | `athenaAsyncQueryDataSupport`                    | Enable async query data support for Athena                                                                                                                                                          | Yes                |
+| `cloudwatchNewRegionsHandler`                    | Refactor of /regions endpoint, no user-facing changes                                                                                                                                               | Yes                |
 | `nestedFolderPicker`                             | Enables the new folder picker to work with nested folders. Requires the nestedFolders feature flag                                                                                                  | Yes                |
 | `accessTokenExpirationCheck`                     | Enable OAuth access_token expiration check and token refresh using the refresh_token                                                                                                                |                    |
 | `emptyDashboardPage`                             | Enable the redesigned user interface of a dashboard page that includes no panels                                                                                                                    | Yes                |

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -47,6 +47,7 @@ export interface FeatureToggles {
   cloudWatchCrossAccountQuerying?: boolean;
   redshiftAsyncQueryDataSupport?: boolean;
   athenaAsyncQueryDataSupport?: boolean;
+  cloudwatchNewRegionsHandler?: boolean;
   showDashboardValidationWarnings?: boolean;
   mysqlAnsiQuotes?: boolean;
   accessControlOnCall?: boolean;

pkg/services/featuremgmt/registry.go

@@ -203,6 +203,13 @@ var (
 			FrontendOnly: true,
 			Owner:        awsDatasourcesSquad,
 		},
+		{
+			Name:        "cloudwatchNewRegionsHandler",
+			Description: "Refactor of /regions endpoint, no user-facing changes",
+			Stage:       FeatureStageGeneralAvailability,
+			Expression:  "true", // enabled by default
+			Owner:       awsDatasourcesSquad,
+		},
 		{
 			Name:        "showDashboardValidationWarnings",
 			Description: "Show warnings when dashboards do not validate against the schema",

pkg/services/featuremgmt/toggles_gen.csv

@@ -28,6 +28,7 @@ entityStore,experimental,@grafana/grafana-app-platform-squad,true,false,false,fa
 cloudWatchCrossAccountQuerying,GA,@grafana/aws-datasources,false,false,false,false
 redshiftAsyncQueryDataSupport,GA,@grafana/aws-datasources,false,false,false,false
 athenaAsyncQueryDataSupport,GA,@grafana/aws-datasources,false,false,false,true
+cloudwatchNewRegionsHandler,GA,@grafana/aws-datasources,false,false,false,false
 showDashboardValidationWarnings,experimental,@grafana/dashboards-squad,false,false,false,false
 mysqlAnsiQuotes,experimental,@grafana/backend-platform,false,false,false,false
 accessControlOnCall,preview,@grafana/grafana-authnz-team,false,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -123,6 +123,10 @@ const (
 	// Enable async query data support for Athena
 	FlagAthenaAsyncQueryDataSupport = "athenaAsyncQueryDataSupport"
 
+	// FlagCloudwatchNewRegionsHandler
+	// Refactor of /regions endpoint, no user-facing changes
+	FlagCloudwatchNewRegionsHandler = "cloudwatchNewRegionsHandler"
+
 	// FlagShowDashboardValidationWarnings
 	// Show warnings when dashboards do not validate against the schema
 	FlagShowDashboardValidationWarnings = "showDashboardValidationWarnings"

pkg/tsdb/cloudwatch/client_factory.go

@@ -49,10 +49,10 @@ var NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatch
 	return cloudwatchlogs.New(sess)
 }
 
-// EC2 client factory.
+// NewEC2Client is a client factory.
 //
 // Stubbable by tests.
-var newEC2Client = func(provider client.ConfigProvider) models.EC2APIProvider {
+var NewEC2Client = func(provider client.ConfigProvider) models.EC2APIProvider {
 	return ec2.New(provider)
 }
 

pkg/tsdb/cloudwatch/cloudwatch.go

@@ -128,14 +128,21 @@ func (e *cloudWatchExecutor) getRequestContext(ctx context.Context, pluginCtx ba
 		r = instance.Settings.Region
 	}
 
+	ec2Client, err := e.getEC2Client(ctx, pluginCtx, defaultRegion)
+	if err != nil {
+		return models.RequestContext{}, err
+	}
+
 	sess, err := e.newSession(ctx, pluginCtx, r)
 	if err != nil {
 		return models.RequestContext{}, err
 	}
+
 	return models.RequestContext{
 		OAMAPIProvider:        NewOAMAPI(sess),
 		MetricsClientProvider: clients.NewMetricsClient(NewMetricsAPI(sess), e.cfg),
 		LogsAPIProvider:       NewLogsAPI(sess),
+		EC2APIProvider:        ec2Client,
 		Settings:              instance.Settings,
 		Features:              e.features,
 	}, nil
@@ -233,6 +240,9 @@ func (e *cloudWatchExecutor) newSession(ctx context.Context, pluginCtx backend.P
 	}
 
 	if region == defaultRegion {
+		if len(instance.Settings.Region) == 0 {
+			return nil, models.ErrMissingRegion
+		}
 		region = instance.Settings.Region
 	}
 
@@ -300,7 +310,7 @@ func (e *cloudWatchExecutor) getEC2Client(ctx context.Context, pluginCtx backend
 		return nil, err
 	}
 
-	return newEC2Client(sess), nil
+	return NewEC2Client(sess), nil
 }
 
 func (e *cloudWatchExecutor) getRGTAClient(ctx context.Context, pluginCtx backend.PluginContext, region string) (resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI,

pkg/tsdb/cloudwatch/cloudwatch_integration_test.go

@@ -7,9 +7,12 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/grafana/grafana-aws-sdk/pkg/awsds"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
@@ -28,17 +31,23 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) {
 	origNewMetricsAPI := NewMetricsAPI
 	origNewOAMAPI := NewOAMAPI
 	origNewLogsAPI := NewLogsAPI
+	origNewEC2Client := NewEC2Client
 	NewOAMAPI = func(sess *session.Session) models.OAMAPIProvider { return nil }
 
 	var logApi mocks.LogsAPI
 	NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider {
 		return &logApi
 	}
-
+	ec2Mock := &mocks.EC2Mock{}
+	ec2Mock.On("DescribeRegions", mock.Anything, mock.Anything).Return(&ec2.DescribeRegionsOutput{}, nil)
+	NewEC2Client = func(provider client.ConfigProvider) models.EC2APIProvider {
+		return ec2Mock
+	}
 	t.Cleanup(func() {
 		NewOAMAPI = origNewOAMAPI
 		NewMetricsAPI = origNewMetricsAPI
 		NewLogsAPI = origNewLogsAPI
+		NewEC2Client = origNewEC2Client
 	})
 
 	var api mocks.FakeMetricsAPI
@@ -46,9 +55,13 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) {
 		return &api
 	}
 
-	im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-		return DataSource{Settings: models.CloudWatchSettings{}}, nil
-	})
+	im := datasource.NewInstanceManager((func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+		return DataSource{Settings: models.CloudWatchSettings{
+			AWSDatasourceSettings: awsds.AWSDatasourceSettings{
+				Region: "us-east-1",
+			},
+		}}, nil
+	}))
 
 	t.Run("Should handle dimension value request and return values from the api", func(t *testing.T) {
 		pageLimit := 100
@@ -239,4 +252,67 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) {
 		require.Nil(t, err)
 		assert.JSONEq(t, `[{"value":{"name":"field1","percent":50}},{"value":{"name":"field2","percent":50}}]`, string(sent.Body))
 	})
+
+	t.Run("Should handle region requests and return regions from the api", func(t *testing.T) {
+		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudwatchNewRegionsHandler, true))
+		req := &backend.CallResourceRequest{
+			Method: "GET",
+			Path:   `/regions`,
+			PluginContext: backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0},
+				PluginID:                   "cloudwatch",
+			},
+		}
+		err := executor.CallResource(context.Background(), req, sender)
+		require.NoError(t, err)
+		sent := sender.Response
+		require.NotNil(t, sent)
+		require.Equal(t, http.StatusOK, sent.Status)
+		require.Nil(t, err)
+		assert.Contains(t, string(sent.Body), `"name":"us-east-1"`)
+	})
+
+	t.Run("Should handle legacy region requests and feature toggle is turned off", func(t *testing.T) {
+		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudwatchNewRegionsHandler, false))
+		req := &backend.CallResourceRequest{
+			Method: "GET",
+			Path:   `/regions`,
+			PluginContext: backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0},
+				PluginID:                   "cloudwatch",
+			},
+		}
+		err := executor.CallResource(context.Background(), req, sender)
+		require.NoError(t, err)
+		sent := sender.Response
+		require.NotNil(t, sent)
+		require.Equal(t, http.StatusOK, sent.Status)
+		require.Nil(t, err)
+		assert.Contains(t, string(sent.Body), `"text":"us-east-1"`)
+	})
+
+	t.Run("Should error for any request when a default region is not selected", func(t *testing.T) {
+		imWithoutDefaultRegion := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+			return DataSource{Settings: models.CloudWatchSettings{
+				AWSDatasourceSettings: awsds.AWSDatasourceSettings{},
+			}}, nil
+		})
+
+		executor := newExecutor(imWithoutDefaultRegion, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures(featuremgmt.FlagCloudwatchNewRegionsHandler, false))
+		req := &backend.CallResourceRequest{
+			Method: "GET",
+			Path:   `/regions`,
+			PluginContext: backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0},
+				PluginID:                   "cloudwatch",
+			},
+		}
+		err := executor.CallResource(context.Background(), req, sender)
+		require.NoError(t, err)
+		sent := sender.Response
+		require.NotNil(t, sent)
+		require.Equal(t, http.StatusBadRequest, sent.Status)
+		require.Nil(t, err)
+		assert.Contains(t, string(sent.Body), "unexpected error missing default region")
+	})
 }

pkg/tsdb/cloudwatch/cloudwatch_test.go

@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/aws/aws-sdk-go/aws"
+	awsclient "github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatch"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
@@ -94,6 +95,7 @@ func Test_CheckHealth(t *testing.T) {
 	origNewMetricsAPI := NewMetricsAPI
 	origNewCWLogsClient := NewCWLogsClient
 	origNewLogsAPI := NewLogsAPI
+
 	t.Cleanup(func() {
 		NewMetricsAPI = origNewMetricsAPI
 		NewCWLogsClient = origNewCWLogsClient
@@ -107,12 +109,16 @@ func Test_CheckHealth(t *testing.T) {
 	NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider {
 		return client
 	}
+	im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+		return DataSource{Settings: models.CloudWatchSettings{
+			AWSDatasourceSettings: awsds.AWSDatasourceSettings{
+				Region: "us-east-1",
+			},
+		}}, nil
+	})
 
 	t.Run("successfully query metrics and logs", func(t *testing.T) {
 		client = fakeCheckHealthClient{}
-		im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-			return DataSource{Settings: models.CloudWatchSettings{}}, nil
-		})
 		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
 
 		resp, err := executor.CheckHealth(context.Background(), &backend.CheckHealthRequest{
@@ -131,9 +137,7 @@ func Test_CheckHealth(t *testing.T) {
 			describeLogGroups: func(input *cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
 				return nil, fmt.Errorf("some logs query error")
 			}}
-		im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-			return DataSource{Settings: models.CloudWatchSettings{}}, nil
-		})
+
 		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
 
 		resp, err := executor.CheckHealth(context.Background(), &backend.CheckHealthRequest{
@@ -152,9 +156,7 @@ func Test_CheckHealth(t *testing.T) {
 			listMetricsPages: func(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error {
 				return fmt.Errorf("some list metrics error")
 			}}
-		im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-			return DataSource{Settings: models.CloudWatchSettings{}}, nil
-		})
+
 		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
 
 		resp, err := executor.CheckHealth(context.Background(), &backend.CheckHealthRequest{
@@ -170,9 +172,7 @@ func Test_CheckHealth(t *testing.T) {
 
 	t.Run("fail to get clients", func(t *testing.T) {
 		client = fakeCheckHealthClient{}
-		im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-			return DataSource{Settings: models.CloudWatchSettings{}}, nil
-		})
+
 		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{getSession: func(c awsds.SessionConfig) (*session.Session, error) {
 			return nil, fmt.Errorf("some sessions error")
 		}}, featuremgmt.WithFeatures())
@@ -194,12 +194,15 @@ func TestQuery_ResourceRequest_DescribeLogGroups_with_CrossAccountQuerying(t *te
 	origNewMetricsAPI := NewMetricsAPI
 	origNewOAMAPI := NewOAMAPI
 	origNewLogsAPI := NewLogsAPI
+	origNewEC2Client := NewEC2Client
 	NewMetricsAPI = func(sess *session.Session) models.CloudWatchMetricsAPIProvider { return nil }
 	NewOAMAPI = func(sess *session.Session) models.OAMAPIProvider { return nil }
+	NewEC2Client = func(provider awsclient.ConfigProvider) models.EC2APIProvider { return nil }
 	t.Cleanup(func() {
 		NewOAMAPI = origNewOAMAPI
 		NewMetricsAPI = origNewMetricsAPI
 		NewLogsAPI = origNewLogsAPI
+		NewEC2Client = origNewEC2Client
 	})
 
 	var logsApi mocks.LogsAPI
@@ -208,7 +211,11 @@ func TestQuery_ResourceRequest_DescribeLogGroups_with_CrossAccountQuerying(t *te
 	}
 
 	im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-		return DataSource{Settings: models.CloudWatchSettings{}}, nil
+		return DataSource{Settings: models.CloudWatchSettings{
+			AWSDatasourceSettings: awsds.AWSDatasourceSettings{
+				Region: "us-east-1",
+			},
+		}}, nil
 	})
 
 	t.Run("maps log group api response to resource response of log-groups", func(t *testing.T) {

pkg/tsdb/cloudwatch/log_actions_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
+	"github.com/grafana/grafana-aws-sdk/pkg/awsds"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
@@ -18,9 +19,9 @@ import (
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/utils"
-	"github.com/stretchr/testify/mock"
 
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
@@ -200,7 +201,11 @@ func TestQuery_StartQuery(t *testing.T) {
 		}
 
 		im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-			return DataSource{Settings: models.CloudWatchSettings{}}, nil
+			return DataSource{Settings: models.CloudWatchSettings{
+				AWSDatasourceSettings: awsds.AWSDatasourceSettings{
+					Region: "us-east-2",
+				},
+			}}, nil
 		})
 
 		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
@@ -253,7 +258,11 @@ func TestQuery_StartQuery(t *testing.T) {
 		}
 
 		im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-			return DataSource{Settings: models.CloudWatchSettings{}}, nil
+			return DataSource{Settings: models.CloudWatchSettings{
+				AWSDatasourceSettings: awsds.AWSDatasourceSettings{
+					Region: "us-east-2",
+				},
+			}}, nil
 		})
 
 		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())

pkg/tsdb/cloudwatch/metric_find_query_test.go

@@ -12,38 +12,43 @@ import (
 	"github.com/aws/aws-sdk-go/service/ec2"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
 	"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
+	"github.com/grafana/grafana-aws-sdk/pkg/awsds"
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
 	"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks"
 	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/utils"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
 
 func TestQuery_Regions(t *testing.T) {
-	origNewEC2Client := newEC2Client
+	origNewEC2Client := NewEC2Client
 	t.Cleanup(func() {
-		newEC2Client = origNewEC2Client
+		NewEC2Client = origNewEC2Client
 	})
 
-	var cli fakeEC2Client
-
-	newEC2Client = func(client.ConfigProvider) models.EC2APIProvider {
-		return cli
+	ec2Mock := &mocks.EC2Mock{}
+	NewEC2Client = func(provider client.ConfigProvider) models.EC2APIProvider {
+		return ec2Mock
 	}
-
 	t.Run("An extra region", func(t *testing.T) {
 		const regionName = "xtra-region"
-		cli = fakeEC2Client{
-			regions: []string{regionName},
-		}
+		ec2Mock.On("DescribeRegions", mock.Anything, mock.Anything).Return(&ec2.DescribeRegionsOutput{
+			Regions: []*ec2.Region{
+				{
+					RegionName: utils.Pointer(regionName),
+				},
+			},
+		}, nil)
 
 		im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-			return DataSource{Settings: models.CloudWatchSettings{}}, nil
+			return DataSource{Settings: models.CloudWatchSettings{AWSDatasourceSettings: awsds.AWSDatasourceSettings{Region: "us-east-2"}}}, nil
 		})
 
 		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
@@ -91,16 +96,16 @@ func buildSortedSliceOfDefaultAndExtraRegions(t *testing.T, regionName string) [
 }
 
 func Test_handleGetRegions_regionCache(t *testing.T) {
-	origNewEC2Client := newEC2Client
+	origNewEC2Client := NewEC2Client
 	t.Cleanup(func() {
-		newEC2Client = origNewEC2Client
+		NewEC2Client = origNewEC2Client
 	})
 	cli := mockEC2Client{}
-	newEC2Client = func(client.ConfigProvider) models.EC2APIProvider {
+	NewEC2Client = func(client.ConfigProvider) models.EC2APIProvider {
 		return &cli
 	}
 	im := datasource.NewInstanceManager(func(ctx context.Context, s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-		return DataSource{Settings: models.CloudWatchSettings{}}, nil
+		return DataSource{Settings: models.CloudWatchSettings{AWSDatasourceSettings: awsds.AWSDatasourceSettings{Region: "us-east-2"}}}, nil
 	})
 
 	t.Run("AWS only called once for multiple calls to handleGetRegions", func(t *testing.T) {
@@ -119,22 +124,21 @@ func Test_handleGetRegions_regionCache(t *testing.T) {
 		cli.AssertNumberOfCalls(t, "DescribeRegions", 1)
 	})
 }
-
 func TestQuery_InstanceAttributes(t *testing.T) {
-	origNewEC2Client := newEC2Client
+	origNewEC2Client := NewEC2Client
 	t.Cleanup(func() {
-		newEC2Client = origNewEC2Client
+		NewEC2Client = origNewEC2Client
 	})
 
-	var cli fakeEC2Client
+	var cli oldEC2Client
 
-	newEC2Client = func(client.ConfigProvider) models.EC2APIProvider {
+	NewEC2Client = func(client.ConfigProvider) models.EC2APIProvider {
 		return cli
 	}
 
 	t.Run("Get instance ID", func(t *testing.T) {
 		const instanceID = "i-12345678"
-		cli = fakeEC2Client{
+		cli = oldEC2Client{
 			reservations: []*ec2.Reservation{
 				{
 					Instances: []*ec2.Instance{
@@ -183,19 +187,19 @@ func TestQuery_InstanceAttributes(t *testing.T) {
 }
 
 func TestQuery_EBSVolumeIDs(t *testing.T) {
-	origNewEC2Client := newEC2Client
+	origNewEC2Client := NewEC2Client
 	t.Cleanup(func() {
-		newEC2Client = origNewEC2Client
+		NewEC2Client = origNewEC2Client
 	})
 
-	var cli fakeEC2Client
+	var cli oldEC2Client
 
-	newEC2Client = func(client.ConfigProvider) models.EC2APIProvider {
+	NewEC2Client = func(client.ConfigProvider) models.EC2APIProvider {
 		return cli
 	}
 
 	t.Run("", func(t *testing.T) {
-		cli = fakeEC2Client{
+		cli = oldEC2Client{
 			reservations: []*ec2.Reservation{
 				{
 					Instances: []*ec2.Instance{

pkg/tsdb/cloudwatch/mocks/regions.go

@@ -0,0 +1,30 @@
+package mocks
+
+import (
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources"
+	"github.com/stretchr/testify/mock"
+)
+
+type RegionsService struct {
+	mock.Mock
+}
+
+func (r *RegionsService) GetRegions() ([]resources.ResourceResponse[resources.Region], error) {
+	args := r.Called()
+	return args.Get(0).(([]resources.ResourceResponse[resources.Region])), args.Error(1)
+}
+
+type EC2Mock struct {
+	mock.Mock
+}
+
+func (e *EC2Mock) DescribeRegions(in *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
+	args := e.Called()
+	return args.Get(0).(*ec2.DescribeRegionsOutput), args.Error(1)
+}
+
+func (e *EC2Mock) DescribeInstancesPages(in *ec2.DescribeInstancesInput, fn func(*ec2.DescribeInstancesOutput, bool) bool) error {
+	args := e.Called(in, fn)
+	return args.Error(0)
+}

pkg/tsdb/cloudwatch/models/api.go

@@ -21,6 +21,7 @@ type RequestContext struct {
 	MetricsClientProvider MetricsClientProvider
 	LogsAPIProvider       CloudWatchLogsAPIProvider
 	OAMAPIProvider        OAMAPIProvider
+	EC2APIProvider        EC2APIProvider
 	Settings              CloudWatchSettings
 	Features              featuremgmt.FeatureToggles
 }
@@ -41,6 +42,10 @@ type AccountsProvider interface {
 	GetAccountsForCurrentUserOrRole() ([]resources.ResourceResponse[resources.Account], error)
 }
 
+type RegionsAPIProvider interface {
+	GetRegions() ([]resources.ResourceResponse[resources.Region], error)
+}
+
 // Clients
 type MetricsClientProvider interface {
 	ListMetricsWithPageLimit(params *cloudwatch.ListMetricsInput) ([]resources.MetricResponse, error)

pkg/tsdb/cloudwatch/models/errors.go

@@ -0,0 +1,7 @@
+package models
+
+import "fmt"
+
+// put misc expected user errors here
+
+var ErrMissingRegion = fmt.Errorf("missing default region")

pkg/tsdb/cloudwatch/models/resources/types.go

@@ -24,6 +24,10 @@ type Account struct {
 	IsMonitoringAccount bool   `json:"isMonitoringAccount"`
 }
 
+type Region struct {
+	Name string `json:"name"`
+}
+
 type Metric struct {
 	Name      string `json:"name"`
 	Namespace string `json:"namespace"`

pkg/tsdb/cloudwatch/resource_handler.go

@@ -15,7 +15,6 @@ import (
 
 func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux {
 	mux := http.NewServeMux()
-	mux.HandleFunc("/regions", handleResourceReq(e.handleGetRegions))
 	mux.HandleFunc("/ebs-volume-ids", handleResourceReq(e.handleGetEbsVolumeIds))
 	mux.HandleFunc("/ec2-instance-attribute", handleResourceReq(e.handleGetEc2InstanceAttribute))
 	mux.HandleFunc("/resource-arns", handleResourceReq(e.handleGetResourceArns))
@@ -28,6 +27,13 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux {
 	mux.HandleFunc("/log-group-fields", routes.ResourceRequestMiddleware(routes.LogGroupFieldsHandler, logger, e.getRequestContext))
 	mux.HandleFunc("/external-id", routes.ResourceRequestMiddleware(routes.ExternalIdHandler, logger, e.getRequestContext))
 
+	// feature is enabled by default, just putting behind a feature flag in case of unexpected bugs
+	if e.features.IsEnabled("cloudwatchNewRegionsHandler") {
+		mux.HandleFunc("/regions", routes.ResourceRequestMiddleware(routes.RegionsHandler, logger, e.getRequestContext))
+	} else {
+		mux.HandleFunc("/regions", handleResourceReq(e.handleGetRegions))
+	}
+
 	// remove this once AWS's Cross Account Observability is supported in GovCloud
 	mux.HandleFunc("/legacy-log-groups", handleResourceReq(e.handleGetLogGroups))
 

pkg/tsdb/cloudwatch/routes/regions.go

@@ -0,0 +1,48 @@
+package routes
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/url"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/services"
+)
+
+const (
+	defaultRegion = "default"
+)
+
+func RegionsHandler(ctx context.Context, pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, parameters url.Values) ([]byte, *models.HttpError) {
+	service, err := newRegionsService(ctx, pluginCtx, reqCtxFactory, defaultRegion)
+	if err != nil {
+		if errors.Is(err, models.ErrMissingRegion) {
+			return nil, models.NewHttpError("Error in Regions Handler when connecting to aws without a default region selection", http.StatusBadRequest, err)
+		}
+		return nil, models.NewHttpError("Error in Regions Handler when connecting to aws", http.StatusInternalServerError, err)
+	}
+
+	regions, err := service.GetRegions()
+	if err != nil {
+		return nil, models.NewHttpError("Error in Regions Handler while fetching regions", http.StatusInternalServerError, err)
+	}
+
+	regionsResponse, err := json.Marshal(regions)
+	if err != nil {
+		return nil, models.NewHttpError("Error in Regions Handler while parsing regions", http.StatusInternalServerError, err)
+	}
+
+	return regionsResponse, nil
+}
+
+var newRegionsService = func(ctx context.Context, pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.RegionsAPIProvider, error) {
+	reqCtx, err := reqCtxFactory(ctx, pluginCtx, region)
+	if err != nil {
+		return nil, err
+	}
+
+	return services.NewRegionsService(reqCtx.EC2APIProvider), nil
+}

pkg/tsdb/cloudwatch/routes/regions_test.go

@@ -0,0 +1,84 @@
+package routes
+
+import (
+	"context"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+)
+
+func TestRegionsRoute(t *testing.T) {
+	origNewRegionsService := newRegionsService
+	t.Cleanup(func() {
+		newRegionsService = origNewRegionsService
+	})
+
+	t.Run("returns 200 and regions", func(t *testing.T) {
+		mockRegionService := mocks.RegionsService{}
+		newRegionsService = func(_ context.Context, pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.RegionsAPIProvider, error) {
+			return &mockRegionService, nil
+		}
+		mockRegionService.On("GetRegions", mock.Anything).Return([]resources.ResourceResponse[resources.Region]{{
+			Value: resources.Region{
+				Name: "us-east-1",
+			},
+		}}, nil).Once()
+
+		rr := httptest.NewRecorder()
+		handler := http.HandlerFunc(ResourceRequestMiddleware(RegionsHandler, logger, nil))
+		req := httptest.NewRequest("GET", `/regions`, nil)
+		handler.ServeHTTP(rr, req)
+
+		assert.Equal(t, http.StatusOK, rr.Code)
+		assert.Contains(t, rr.Body.String(), "us-east-1")
+	})
+
+	t.Run("returns 400 when the service returns a missing region error", func(t *testing.T) {
+		newRegionsService = func(_ context.Context, pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.RegionsAPIProvider, error) {
+			return nil, models.ErrMissingRegion
+		}
+		rr := httptest.NewRecorder()
+		handler := http.HandlerFunc(ResourceRequestMiddleware(RegionsHandler, logger, nil))
+		req := httptest.NewRequest("GET", `/regions`, nil)
+		handler.ServeHTTP(rr, req)
+
+		assert.Equal(t, http.StatusBadRequest, rr.Code)
+		assert.Contains(t, rr.Body.String(), "Error in Regions Handler when connecting to aws without a default region selection: missing default region")
+	})
+
+	t.Run("returns 500 when the service returns an unexpected error", func(t *testing.T) {
+		newRegionsService = func(_ context.Context, pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.RegionsAPIProvider, error) {
+			return nil, errors.New("something unexpected happened")
+		}
+		rr := httptest.NewRecorder()
+		handler := http.HandlerFunc(ResourceRequestMiddleware(RegionsHandler, logger, nil))
+		req := httptest.NewRequest("GET", `/regions`, nil)
+		handler.ServeHTTP(rr, req)
+
+		assert.Equal(t, http.StatusInternalServerError, rr.Code)
+		assert.Contains(t, rr.Body.String(), "Error in Regions Handler when connecting to aws: something unexpected happened")
+	})
+
+	t.Run("returns 500 when get regions returns an error", func(t *testing.T) {
+		mockRegionService := mocks.RegionsService{}
+		newRegionsService = func(_ context.Context, pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.RegionsAPIProvider, error) {
+			return &mockRegionService, nil
+		}
+		mockRegionService.On("GetRegions", mock.Anything).Return([]resources.ResourceResponse[resources.Region](nil), errors.New("aws is having some kind of outage")).Once()
+		rr := httptest.NewRecorder()
+		handler := http.HandlerFunc(ResourceRequestMiddleware(RegionsHandler, logger, nil))
+		req := httptest.NewRequest("GET", `/regions`, nil)
+		handler.ServeHTTP(rr, req)
+
+		assert.Equal(t, http.StatusInternalServerError, rr.Code)
+		assert.Contains(t, rr.Body.String(), "Error in Regions Handler while fetching regions: aws is having some kind of outage")
+	})
+}

pkg/tsdb/cloudwatch/services/regions.go

@@ -0,0 +1,55 @@
+package services
+
+import (
+	"sort"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/constants"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources"
+)
+
+type RegionsService struct {
+	models.EC2APIProvider
+}
+
+func NewRegionsService(ec2client models.EC2APIProvider) models.RegionsAPIProvider {
+	return &RegionsService{
+		ec2client,
+	}
+}
+
+func mergeEC2RegionsAndConstantRegions(regions map[string]struct{}, ec2Regions []*ec2.Region) {
+	for _, region := range ec2Regions {
+		if _, ok := regions[*region.RegionName]; !ok {
+			regions[*region.RegionName] = struct{}{}
+		}
+	}
+}
+
+func (r *RegionsService) GetRegions() ([]resources.ResourceResponse[resources.Region], error) {
+	regions := constants.Regions()
+
+	result := make([]resources.ResourceResponse[resources.Region], 0)
+
+	ec2Regions, err := r.DescribeRegions(&ec2.DescribeRegionsInput{})
+	if err != nil {
+		return nil, err
+	}
+
+	mergeEC2RegionsAndConstantRegions(regions, ec2Regions.Regions)
+
+	for region := range regions {
+		result = append(result, resources.ResourceResponse[resources.Region]{
+			Value: resources.Region{
+				Name: region,
+			},
+		})
+	}
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Value.Name < result[j].Value.Name
+	})
+
+	return result, nil
+}

pkg/tsdb/cloudwatch/services/regions_test.go

@@ -0,0 +1,44 @@
+package services
+
+import (
+	"testing"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources"
+	"github.com/grafana/grafana/pkg/tsdb/cloudwatch/utils"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestRegions(t *testing.T) {
+	t.Run("returns regions from the api and merges them with default regions", func(t *testing.T) {
+		mockRegions := &ec2.DescribeRegionsOutput{
+			Regions: []*ec2.Region{
+				{
+					RegionName: utils.Pointer("earth-1"),
+				},
+			},
+		}
+		ec2Mock := &mocks.EC2Mock{}
+		ec2Mock.On("DescribeRegions").Return(mockRegions, nil)
+		regions, err := NewRegionsService(ec2Mock).GetRegions()
+		assert.NoError(t, err)
+		assert.Contains(t, regions, resources.ResourceResponse[resources.Region]{
+			Value: resources.Region{
+				Name: "us-east-2",
+			},
+		})
+		assert.Contains(t, regions, resources.ResourceResponse[resources.Region]{
+			Value: resources.Region{
+				Name: "earth-1",
+			},
+		})
+	})
+
+	t.Run("forwards error if DescribeRegions errors out", func(t *testing.T) {
+		ec2Mock := &mocks.EC2Mock{}
+		ec2Mock.On("DescribeRegions").Return((*ec2.DescribeRegionsOutput)(nil), assert.AnError)
+		_, err := NewRegionsService(ec2Mock).GetRegions()
+		assert.Error(t, err)
+	})
+}

pkg/tsdb/cloudwatch/test_utils.go

@@ -135,14 +135,15 @@ func (c *mockEC2Client) DescribeInstancesPages(in *ec2.DescribeInstancesInput, f
 	return args.Error(0)
 }
 
-type fakeEC2Client struct {
+// Please use mockEC2Client above, we are slowly migrating towards using testify's mocks only
+type oldEC2Client struct {
 	ec2iface.EC2API
 
 	regions      []string
 	reservations []*ec2.Reservation
 }
 
-func (c fakeEC2Client) DescribeRegions(*ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
+func (c oldEC2Client) DescribeRegions(*ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
 	regions := []*ec2.Region{}
 	for _, region := range c.regions {
 		regions = append(regions, &ec2.Region{
@@ -154,7 +155,7 @@ func (c fakeEC2Client) DescribeRegions(*ec2.DescribeRegionsInput) (*ec2.Describe
 	}, nil
 }
 
-func (c fakeEC2Client) DescribeInstancesPages(in *ec2.DescribeInstancesInput,
+func (c oldEC2Client) DescribeInstancesPages(in *ec2.DescribeInstancesInput,
 	fn func(*ec2.DescribeInstancesOutput, bool) bool) error {
 	reservations := []*ec2.Reservation{}
 	for _, r := range c.reservations {

public/app/plugins/datasource/cloudwatch/__mocks__/ResourcesAPI.ts

@@ -13,7 +13,7 @@ export function setupMockedResourcesAPI({
   getMock,
 }: {
   getMock?: jest.Mock;
-  response?: Array<{ text: string; label: string; value: string }>;
+  response?: unknown;
   variables?: CustomVariableModel[];
   mockGetVariableName?: boolean;
 } = {}) {

public/app/plugins/datasource/cloudwatch/resources/ResourceAPI.test.ts

@@ -1,3 +1,5 @@
+import { config } from '@grafana/runtime';
+
 import { setupMockedResourcesAPI } from '../__mocks__/ResourcesAPI';
 
 describe('ResourcesAPI', () => {
@@ -118,4 +120,109 @@ describe('ResourcesAPI', () => {
       ]);
     });
   });
+
+  const originalFeatureToggleValue = config.featureToggles.cloudwatchNewRegionsHandler;
+
+  describe('getRegions', () => {
+    afterEach(() => {
+      config.featureToggles.cloudwatchNewRegionsHandler = originalFeatureToggleValue;
+    });
+    it('should return regions as an array of options when using legacy regions route', async () => {
+      config.featureToggles.cloudwatchNewRegionsHandler = false;
+      const response = Promise.resolve([
+        {
+          text: 'US East (Ohio)',
+          value: 'us-east-2',
+          label: 'US East (Ohio)',
+        },
+        {
+          text: 'US East (N. Virginia)',
+          value: 'us-east-1',
+          label: 'US East (N. Virginia)',
+        },
+        {
+          text: 'US West (N. California)',
+          value: 'us-west-1',
+          label: 'US West (N. California)',
+        },
+      ]);
+
+      const { api } = setupMockedResourcesAPI({ response });
+      const expectedRegions = [
+        {
+          text: 'default',
+          value: 'default',
+          label: 'default',
+        },
+        {
+          text: 'US East (Ohio)',
+          value: 'us-east-2',
+          label: 'US East (Ohio)',
+        },
+        {
+          text: 'US East (N. Virginia)',
+          value: 'us-east-1',
+          label: 'US East (N. Virginia)',
+        },
+        {
+          text: 'US West (N. California)',
+          value: 'us-west-1',
+          label: 'US West (N. California)',
+        },
+      ];
+
+      const regions = await api.getRegions();
+
+      expect(regions).toEqual(expectedRegions);
+    });
+
+    it('should return regions as an array of options when using new regions route', async () => {
+      config.featureToggles.cloudwatchNewRegionsHandler = true;
+      const response = Promise.resolve([
+        {
+          value: {
+            name: 'us-east-2',
+          },
+        },
+        {
+          value: {
+            name: 'us-east-1',
+          },
+        },
+        {
+          value: {
+            name: 'us-west-1',
+          },
+        },
+      ]);
+
+      const { api } = setupMockedResourcesAPI({ response });
+      const expectedRegions = [
+        {
+          text: 'default',
+          value: 'default',
+          label: 'default',
+        },
+        {
+          text: 'us-east-2',
+          value: 'us-east-2',
+          label: 'us-east-2',
+        },
+        {
+          text: 'us-east-1',
+          value: 'us-east-1',
+          label: 'us-east-1',
+        },
+        {
+          text: 'us-west-1',
+          value: 'us-west-1',
+          label: 'us-west-1',
+        },
+      ];
+
+      const regions = await api.getRegions();
+
+      expect(regions).toEqual(expectedRegions);
+    });
+  });
 });

public/app/plugins/datasource/cloudwatch/resources/ResourcesAPI.ts

@@ -1,7 +1,7 @@
 import { memoize } from 'lodash';
 
 import { DataSourceInstanceSettings, SelectableValue } from '@grafana/data';
-import { getBackendSrv } from '@grafana/runtime';
+import { getBackendSrv, config } from '@grafana/runtime';
 import { TemplateSrv } from 'app/features/templating/template_srv';
 
 import { CloudWatchRequest } from '../query-runner/CloudWatchRequest';
@@ -19,6 +19,7 @@ import {
   GetDimensionValuesRequest,
   MetricResponse,
   SelectableResourceValue,
+  RegionResponse,
 } from './types';
 
 export class ResourcesAPI extends CloudWatchRequest {
@@ -51,11 +52,23 @@ export class ResourcesAPI extends CloudWatchRequest {
       .catch(() => false);
   }
 
-  getRegions() {
-    return this.memoizedGetRequest<SelectableResourceValue[]>('regions').then((regions) => [
-      { label: 'default', value: 'default', text: 'default' },
-      ...regions.filter((r) => r.value),
-    ]);
+  getRegions(): Promise<SelectableResourceValue[]> {
+    if (!config.featureToggles.cloudwatchNewRegionsHandler) {
+      return this.memoizedGetRequest<SelectableResourceValue[]>('regions').then((regions) => [
+        { label: 'default', value: 'default', text: 'default' },
+        ...regions.filter((r) => r.value),
+      ]);
+    }
+    return this.memoizedGetRequest<Array<ResourceResponse<RegionResponse>>>('regions').then((regions) => {
+      return [
+        { label: 'default', value: 'default', text: 'default' },
+        ...regions.map((r) => ({
+          label: r.value.name,
+          value: r.value.name,
+          text: r.value.name,
+        })),
+      ].filter((r) => r.value);
+    });
   }
 
   getNamespaces() {

public/app/plugins/datasource/cloudwatch/resources/types.ts

@@ -59,6 +59,10 @@ export interface MetricResponse {
   namespace: string;
 }
 
+export interface RegionResponse {
+  name: string;
+}
+
 export interface SelectableResourceValue extends SelectableValue<string> {
   text: string;
 }