Cloudwatch: Refactor - move describe all log groups to call resource handler (#55582)

* refactor backend

* move describe log groups to api file

* fix lint issue

pkg/tsdb/cloudwatch/cloudwatch_test.go

@@ -248,6 +248,85 @@ func Test_executeLogAlertQuery(t *testing.T) {
 	})
 }
 
+func TestQuery_ResourceRequest_DescribeAllLogGroups(t *testing.T) {
+	origNewCWLogsClient := NewCWLogsClient
+	t.Cleanup(func() {
+		NewCWLogsClient = origNewCWLogsClient
+	})
+
+	var cli fakeCWLogsClient
+
+	NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
+		return &cli
+	}
+
+	im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+		return datasourceInfo{}, nil
+	})
+
+	executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
+	sender := &mockedCallResourceResponseSenderForOauth{}
+
+	t.Run("multiple batches", func(t *testing.T) {
+		token := "foo"
+		cli = fakeCWLogsClient{
+			logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
+				{
+					LogGroups: []*cloudwatchlogs.LogGroup{
+						{
+							LogGroupName: aws.String("group_a"),
+						},
+						{
+							LogGroupName: aws.String("group_b"),
+						},
+						{
+							LogGroupName: aws.String("group_c"),
+						},
+					},
+					NextToken: &token,
+				},
+				{
+					LogGroups: []*cloudwatchlogs.LogGroup{
+						{
+							LogGroupName: aws.String("group_x"),
+						},
+						{
+							LogGroupName: aws.String("group_y"),
+						},
+						{
+							LogGroupName: aws.String("group_z"),
+						},
+					},
+				},
+			},
+		}
+
+		req := &backend.CallResourceRequest{
+			Method: "GET",
+			Path:   "/all-log-groups?limit=50",
+			PluginContext: backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{
+					ID: 0,
+				},
+				PluginID: "cloudwatch",
+			},
+		}
+		err := executor.CallResource(context.Background(), req, sender)
+		require.NoError(t, err)
+		sent := sender.Response
+		require.NotNil(t, sent)
+		require.Equal(t, http.StatusOK, sent.Status)
+
+		suggestDataResponse := []suggestData{}
+		err = json.Unmarshal(sent.Body, &suggestDataResponse)
+		require.Nil(t, err)
+
+		assert.Equal(t, stringsToSuggestData([]string{
+			"group_a", "group_b", "group_c", "group_x", "group_y", "group_z",
+		}), suggestDataResponse)
+	})
+}
+
 func TestQuery_ResourceRequest_DescribeLogGroups(t *testing.T) {
 	origNewCWLogsClient := NewCWLogsClient
 	t.Cleanup(func() {
@@ -305,8 +384,7 @@ func TestQuery_ResourceRequest_DescribeLogGroups(t *testing.T) {
 		err = json.Unmarshal(sent.Body, &suggestDataResponse)
 		require.Nil(t, err)
 
-		assert.Equal(t, []suggestData{
+		assert.Equal(t, stringsToSuggestData([]string{"group_a", "group_b", "group_c"}), suggestDataResponse)
-			{Text: "group_a", Value: "group_a", Label: "group_a"}, {Text: "group_b", Value: "group_b", Label: "group_b"}, {Text: "group_c", Value: "group_c", Label: "group_c"}}, suggestDataResponse)
 	})
 
 	t.Run("Should call api with LogGroupNamePrefix if passed in resource call", func(t *testing.T) {
@@ -384,3 +462,11 @@ func TestQuery_ResourceRequest_DescribeLogGroups(t *testing.T) {
 		}, cli.calls.describeLogGroups)
 	})
 }
+
+func stringsToSuggestData(values []string) []suggestData {
+	suggestDataArray := make([]suggestData, 0)
+	for _, v := range values {
+		suggestDataArray = append(suggestDataArray, suggestData{Text: v, Value: v, Label: v})
+	}
+	return suggestDataArray
+}

pkg/tsdb/cloudwatch/log_actions.go

@@ -126,8 +126,6 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, model LogQuer
 
 	var data *data.Frame = nil
 	switch model.SubType {
-	case "DescribeAllLogGroups":
-		data, err = e.handleDescribeAllLogGroups(ctx, logsClient, model)
 	case "GetLogGroupFields":
 		data, err = e.handleGetLogGroupFields(ctx, logsClient, model, query.RefID)
 	case "StartQuery":
@@ -201,40 +199,6 @@ func (e *cloudWatchExecutor) handleGetLogEvents(ctx context.Context, logsClient
 	return data.NewFrame("logEvents", timestampField, messageField), nil
 }
 
-func (e *cloudWatchExecutor) handleDescribeAllLogGroups(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson) (*data.Frame, error) {
-	var namePrefix, nextToken *string
-	if len(parameters.LogGroupNamePrefix) != 0 {
-		namePrefix = aws.String(parameters.LogGroupNamePrefix)
-	}
-
-	var response *cloudwatchlogs.DescribeLogGroupsOutput
-	var err error
-	logGroupNames := []*string{}
-	for {
-		response, err = logsClient.DescribeLogGroupsWithContext(ctx, &cloudwatchlogs.DescribeLogGroupsInput{
-			LogGroupNamePrefix: namePrefix,
-			NextToken:          nextToken,
-			Limit:              aws.Int64(defaultLogGroupLimit),
-		})
-		if err != nil || response == nil {
-			return nil, err
-		}
-
-		for _, logGroup := range response.LogGroups {
-			logGroupNames = append(logGroupNames, logGroup.LogGroupName)
-		}
-
-		if response.NextToken == nil {
-			break
-		}
-		nextToken = response.NextToken
-	}
-
-	groupNamesField := data.NewField("logGroupName", nil, logGroupNames)
-	frame := data.NewFrame("logGroups", groupNamesField)
-	return frame, nil
-}
-
 func (e *cloudWatchExecutor) executeStartQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI,
 	parameters LogQueryJson, timeRange backend.TimeRange) (*cloudwatchlogs.StartQueryOutput, error) {
 	startTime := timeRange.From

pkg/tsdb/cloudwatch/log_actions_test.go

@@ -106,92 +106,6 @@ func TestQuery_GetLogEvents(t *testing.T) {
 	}
 }
 
-func TestQuery_DescribeAllLogGroups(t *testing.T) {
-	origNewCWLogsClient := NewCWLogsClient
-	t.Cleanup(func() {
-		NewCWLogsClient = origNewCWLogsClient
-	})
-
-	var cli fakeCWLogsClient
-
-	NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
-		return &cli
-	}
-
-	t.Run("multiple batches", func(t *testing.T) {
-		token := "foo"
-		cli = fakeCWLogsClient{
-			logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
-				{
-					LogGroups: []*cloudwatchlogs.LogGroup{
-						{
-							LogGroupName: aws.String("group_a"),
-						},
-						{
-							LogGroupName: aws.String("group_b"),
-						},
-						{
-							LogGroupName: aws.String("group_c"),
-						},
-					},
-					NextToken: &token,
-				},
-				{
-					LogGroups: []*cloudwatchlogs.LogGroup{
-						{
-							LogGroupName: aws.String("group_x"),
-						},
-						{
-							LogGroupName: aws.String("group_y"),
-						},
-						{
-							LogGroupName: aws.String("group_z"),
-						},
-					},
-				},
-			},
-		}
-
-		im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
-			return datasourceInfo{}, nil
-		})
-
-		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
-		resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
-			PluginContext: backend.PluginContext{
-				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
-			},
-			Queries: []backend.DataQuery{
-				{
-					JSON: json.RawMessage(`{
-						"type":    "logAction",
-						"subtype": "DescribeAllLogGroups",
-						"limit":   50
-					}`),
-				},
-			},
-		})
-		require.NoError(t, err)
-		require.NotNil(t, resp)
-
-		assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
-			"": backend.DataResponse{
-				Frames: data.Frames{
-					&data.Frame{
-						Name: "logGroups",
-						Fields: []*data.Field{
-							data.NewField("logGroupName", nil, []*string{
-								aws.String("group_a"), aws.String("group_b"), aws.String("group_c"), aws.String("group_x"), aws.String("group_y"), aws.String("group_z"),
-							}),
-						},
-					},
-				},
-			},
-		},
-		}, resp)
-	})
-}
-
 func TestQuery_GetLogGroupFields(t *testing.T) {
 	origNewCWLogsClient := NewCWLogsClient
 	t.Cleanup(func() {

pkg/tsdb/cloudwatch/metric_find_query.go

@@ -685,6 +685,42 @@ func (e *cloudWatchExecutor) handleGetLogGroups(pluginCtx backend.PluginContext,
 
 	return result, nil
 }
+func (e *cloudWatchExecutor) handleGetAllLogGroups(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
+	var nextToken *string
+
+	logGroupNamePrefix := parameters.Get("logGroupNamePrefix")
+
+	var err error
+	logsClient, err := e.getCWLogsClient(pluginCtx, parameters.Get("region"))
+	if err != nil {
+		return nil, err
+	}
+
+	var response *cloudwatchlogs.DescribeLogGroupsOutput
+	result := make([]suggestData, 0)
+	for {
+		response, err = logsClient.DescribeLogGroups(&cloudwatchlogs.DescribeLogGroupsInput{
+			LogGroupNamePrefix: aws.String(logGroupNamePrefix),
+			NextToken:          nextToken,
+			Limit:              aws.Int64(defaultLogGroupLimit),
+		})
+		if err != nil || response == nil {
+			return nil, err
+		}
+
+		for _, logGroup := range response.LogGroups {
+			logGroupName := *logGroup.LogGroupName
+			result = append(result, suggestData{Text: logGroupName, Value: logGroupName, Label: logGroupName})
+		}
+
+		if response.NextToken == nil {
+			break
+		}
+		nextToken = response.NextToken
+	}
+
+	return result, nil
+}
 
 func isDuplicate(nameList []string, target string) bool {
 	for _, name := range nameList {

pkg/tsdb/cloudwatch/resource_handler.go

@@ -22,6 +22,7 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux {
 	mux.HandleFunc("/ec2-instance-attribute", handleResourceReq(e.handleGetEc2InstanceAttribute))
 	mux.HandleFunc("/resource-arns", handleResourceReq(e.handleGetResourceArns))
 	mux.HandleFunc("/log-groups", handleResourceReq(e.handleGetLogGroups))
+	mux.HandleFunc("/all-log-groups", handleResourceReq(e.handleGetAllLogGroups))
 	return mux
 }
 

public/app/plugins/datasource/cloudwatch/api.ts

@@ -39,6 +39,13 @@ export class CloudWatchAPI extends CloudWatchRequest {
     });
   }
 
+  async describeAllLogGroups(params: DescribeLogGroupsRequest) {
+    return this.resourceRequest('all-log-groups', {
+      ...params,
+      region: this.templateSrv.replace(this.getActualRegion(params.region)),
+    });
+  }
+
   async getMetrics(namespace: string | undefined, region?: string) {
     if (!namespace) {
       return [];

public/app/plugins/datasource/cloudwatch/datasource.ts

@@ -65,7 +65,7 @@ export class CloudWatchDatasource
     this.metricsQueryRunner = new CloudWatchMetricsQueryRunner(instanceSettings, templateSrv);
     this.logsQueryRunner = new CloudWatchLogsQueryRunner(instanceSettings, templateSrv, timeSrv);
     this.annotationQueryRunner = new CloudWatchAnnotationQueryRunner(instanceSettings, templateSrv);
-    this.variables = new CloudWatchVariableSupport(this.api, this.logsQueryRunner);
+    this.variables = new CloudWatchVariableSupport(this.api);
     this.annotations = CloudWatchAnnotationSupport;
   }
 

public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts

@@ -425,13 +425,6 @@ export class CloudWatchLogsQueryRunner extends CloudWatchRequest {
     };
   };
 
-  async describeAllLogGroups(params: DescribeLogGroupsRequest): Promise<string[]> {
-    const dataFrames = await lastValueFrom(this.makeLogActionRequest('DescribeAllLogGroups', [params]));
-
-    const logGroupNames = dataFrames[0]?.fields[0]?.values.toArray() ?? [];
-    return logGroupNames;
-  }
-
   async getLogGroupFields(params: GetLogGroupFieldsRequest): Promise<GetLogGroupFieldsResponse> {
     const dataFrames = await lastValueFrom(this.makeLogActionRequest('GetLogGroupFields', [params]));
 

public/app/plugins/datasource/cloudwatch/variables.test.ts

@@ -1,3 +1,5 @@
+import { toOption } from '@grafana/data';
+
 import { dimensionVariable, labelsVariable, setupMockedDataSource } from './__mocks__/CloudWatchDataSource';
 import { VariableQuery, VariableQueryType } from './types';
 import { CloudWatchVariableSupport } from './variables';
@@ -19,13 +21,13 @@ mock.datasource.api.getRegions = jest.fn().mockResolvedValue([{ label: 'a', valu
 mock.datasource.api.getNamespaces = jest.fn().mockResolvedValue([{ label: 'b', value: 'b' }]);
 mock.datasource.api.getMetrics = jest.fn().mockResolvedValue([{ label: 'c', value: 'c' }]);
 mock.datasource.api.getDimensionKeys = jest.fn().mockResolvedValue([{ label: 'd', value: 'd' }]);
-mock.datasource.logsQueryRunner.describeAllLogGroups = jest.fn().mockResolvedValue(['a', 'b']);
+mock.datasource.api.describeAllLogGroups = jest.fn().mockResolvedValue(['a', 'b'].map(toOption));
 const getDimensionValues = jest.fn().mockResolvedValue([{ label: 'e', value: 'e' }]);
 const getEbsVolumeIds = jest.fn().mockResolvedValue([{ label: 'f', value: 'f' }]);
 const getEc2InstanceAttribute = jest.fn().mockResolvedValue([{ label: 'g', value: 'g' }]);
 const getResourceARNs = jest.fn().mockResolvedValue([{ label: 'h', value: 'h' }]);
 
-const variables = new CloudWatchVariableSupport(mock.datasource.api, mock.datasource.logsQueryRunner);
+const variables = new CloudWatchVariableSupport(mock.datasource.api);
 
 describe('variables', () => {
   it('should run regions', async () => {

public/app/plugins/datasource/cloudwatch/variables.ts

@@ -7,12 +7,11 @@ import { CloudWatchAPI } from './api';
 import { VariableQueryEditor } from './components/VariableQueryEditor/VariableQueryEditor';
 import { CloudWatchDatasource } from './datasource';
 import { migrateVariableQuery } from './migrations/variableQueryMigrations';
-import { CloudWatchLogsQueryRunner } from './query-runner/CloudWatchLogsQueryRunner';
 import { standardStatistics } from './standardStatistics';
 import { VariableQuery, VariableQueryType } from './types';
 
 export class CloudWatchVariableSupport extends CustomVariableSupport<CloudWatchDatasource, VariableQuery> {
-  constructor(private readonly api: CloudWatchAPI, private readonly logsQueryRunner: CloudWatchLogsQueryRunner) {
+  constructor(private readonly api: CloudWatchAPI) {
     super();
     this.query = this.query.bind(this);
   }
@@ -55,13 +54,13 @@ export class CloudWatchVariableSupport extends CustomVariableSupport<CloudWatchD
   }
 
   async handleLogGroupsQuery({ region, logGroupPrefix }: VariableQuery) {
-    const logGroups: string[] = await this.logsQueryRunner.describeAllLogGroups({
+    const logGroups = await this.api.describeAllLogGroups({
       region,
       logGroupNamePrefix: logGroupPrefix,
     });
     return logGroups.map((s) => ({
-      text: s,
+      text: s.value,
-      value: s,
+      value: s.value,
       expandable: true,
     }));
   }