CloudWatch: Log group variable should get all log groups (#54062)

pkg/tsdb/cloudwatch/log_actions.go

@@ -19,8 +19,8 @@ import (
 
 const (
 	limitExceededException = "LimitExceededException"
-	defaultLimit           = int64(10)
+	defaultEventLimit      = int64(10)
-	logGroupDefaultLimit   = int64(50)
+	defaultLogGroupLimit   = int64(50)
 )
 
 type AWSError struct {
@@ -128,6 +128,8 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, model LogQuer
 	switch model.SubType {
 	case "DescribeLogGroups":
 		data, err = e.handleDescribeLogGroups(ctx, logsClient, model)
+	case "DescribeAllLogGroups":
+		data, err = e.handleDescribeAllLogGroups(ctx, logsClient, model)
 	case "GetLogGroupFields":
 		data, err = e.handleGetLogGroupFields(ctx, logsClient, model, query.RefID)
 	case "StartQuery":
@@ -148,7 +150,7 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, model LogQuer
 
 func (e *cloudWatchExecutor) handleGetLogEvents(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI,
 	parameters LogQueryJson) (*data.Frame, error) {
-	limit := defaultLimit
+	limit := defaultEventLimit
 	if parameters.Limit != nil && *parameters.Limit > 0 {
 		limit = *parameters.Limit
 	}
@@ -203,7 +205,7 @@ func (e *cloudWatchExecutor) handleGetLogEvents(ctx context.Context, logsClient
 
 func (e *cloudWatchExecutor) handleDescribeLogGroups(ctx context.Context,
 	logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson) (*data.Frame, error) {
-	logGroupLimit := logGroupDefaultLimit
+	logGroupLimit := defaultLogGroupLimit
 	if parameters.Limit != nil && *parameters.Limit != 0 {
 		logGroupLimit = *parameters.Limit
 	}
@@ -235,6 +237,40 @@ func (e *cloudWatchExecutor) handleDescribeLogGroups(ctx context.Context,
 	return frame, nil
 }
 
+func (e *cloudWatchExecutor) handleDescribeAllLogGroups(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson) (*data.Frame, error) {
+	var namePrefix, nextToken *string
+	if len(parameters.LogGroupNamePrefix) != 0 {
+		namePrefix = aws.String(parameters.LogGroupNamePrefix)
+	}
+
+	var response *cloudwatchlogs.DescribeLogGroupsOutput
+	var err error
+	logGroupNames := []*string{}
+	for {
+		response, err = logsClient.DescribeLogGroupsWithContext(ctx, &cloudwatchlogs.DescribeLogGroupsInput{
+			LogGroupNamePrefix: namePrefix,
+			NextToken:          nextToken,
+			Limit:              aws.Int64(defaultLogGroupLimit),
+		})
+		if err != nil || response == nil {
+			return nil, err
+		}
+
+		for _, logGroup := range response.LogGroups {
+			logGroupNames = append(logGroupNames, logGroup.LogGroupName)
+		}
+
+		if response.NextToken == nil {
+			break
+		}
+		nextToken = response.NextToken
+	}
+
+	groupNamesField := data.NewField("logGroupName", nil, logGroupNames)
+	frame := data.NewFrame("logGroups", groupNamesField)
+	return frame, nil
+}
+
 func (e *cloudWatchExecutor) executeStartQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI,
 	parameters LogQueryJson, timeRange backend.TimeRange) (*cloudwatchlogs.StartQueryOutput, error) {
 	startTime := timeRange.From

pkg/tsdb/cloudwatch/log_actions_test.go

@@ -120,8 +120,8 @@ func TestQuery_DescribeLogGroups(t *testing.T) {
 
 	t.Run("Empty log group name prefix", func(t *testing.T) {
 		cli = fakeCWLogsClient{
-			logGroups: cloudwatchlogs.DescribeLogGroupsOutput{
+			logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
-				LogGroups: []*cloudwatchlogs.LogGroup{
+				{LogGroups: []*cloudwatchlogs.LogGroup{
 					{
 						LogGroupName: aws.String("group_a"),
 					},
@@ -131,7 +131,7 @@ func TestQuery_DescribeLogGroups(t *testing.T) {
 					{
 						LogGroupName: aws.String("group_c"),
 					},
-				},
+				}},
 			},
 		}
 
@@ -176,8 +176,8 @@ func TestQuery_DescribeLogGroups(t *testing.T) {
 
 	t.Run("Non-empty log group name prefix", func(t *testing.T) {
 		cli = fakeCWLogsClient{
-			logGroups: cloudwatchlogs.DescribeLogGroupsOutput{
+			logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
-				LogGroups: []*cloudwatchlogs.LogGroup{
+				{LogGroups: []*cloudwatchlogs.LogGroup{
 					{
 						LogGroupName: aws.String("group_a"),
 					},
@@ -187,7 +187,7 @@ func TestQuery_DescribeLogGroups(t *testing.T) {
 					{
 						LogGroupName: aws.String("group_c"),
 					},
-				},
+				}},
 			},
 		}
 
@@ -233,6 +233,92 @@ func TestQuery_DescribeLogGroups(t *testing.T) {
 	})
 }
 
+func TestQuery_DescribeAllLogGroups(t *testing.T) {
+	origNewCWLogsClient := NewCWLogsClient
+	t.Cleanup(func() {
+		NewCWLogsClient = origNewCWLogsClient
+	})
+
+	var cli fakeCWLogsClient
+
+	NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
+		return &cli
+	}
+
+	t.Run("multiple batches", func(t *testing.T) {
+		token := "foo"
+		cli = fakeCWLogsClient{
+			logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{
+				{
+					LogGroups: []*cloudwatchlogs.LogGroup{
+						{
+							LogGroupName: aws.String("group_a"),
+						},
+						{
+							LogGroupName: aws.String("group_b"),
+						},
+						{
+							LogGroupName: aws.String("group_c"),
+						},
+					},
+					NextToken: &token,
+				},
+				{
+					LogGroups: []*cloudwatchlogs.LogGroup{
+						{
+							LogGroupName: aws.String("group_x"),
+						},
+						{
+							LogGroupName: aws.String("group_y"),
+						},
+						{
+							LogGroupName: aws.String("group_z"),
+						},
+					},
+				},
+			},
+		}
+
+		im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
+			return datasourceInfo{}, nil
+		})
+
+		executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
+		resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
+			PluginContext: backend.PluginContext{
+				DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
+			},
+			Queries: []backend.DataQuery{
+				{
+					JSON: json.RawMessage(`{
+						"type":    "logAction",
+						"subtype": "DescribeAllLogGroups",
+						"limit":   50
+					}`),
+				},
+			},
+		})
+		require.NoError(t, err)
+		require.NotNil(t, resp)
+
+		assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
+			"": backend.DataResponse{
+				Frames: data.Frames{
+					&data.Frame{
+						Name: "logGroups",
+						Fields: []*data.Field{
+							data.NewField("logGroupName", nil, []*string{
+								aws.String("group_a"), aws.String("group_b"), aws.String("group_c"), aws.String("group_x"), aws.String("group_y"), aws.String("group_z"),
+							}),
+						},
+					},
+				},
+			},
+		},
+		}, resp)
+	})
+}
+
 func TestQuery_GetLogGroupFields(t *testing.T) {
 	origNewCWLogsClient := NewCWLogsClient
 	t.Cleanup(func() {

pkg/tsdb/cloudwatch/utils_test.go

@@ -23,9 +23,11 @@ type fakeCWLogsClient struct {
 
 	calls logsQueryCalls
 
-	logGroups      cloudwatchlogs.DescribeLogGroupsOutput
+	logGroups      []cloudwatchlogs.DescribeLogGroupsOutput
 	logGroupFields cloudwatchlogs.GetLogGroupFieldsOutput
 	queryResults   cloudwatchlogs.GetQueryResultsOutput
+
+	logGroupsIndex int
 }
 
 type logsQueryCalls struct {
@@ -52,7 +54,9 @@ func (m *fakeCWLogsClient) StopQueryWithContext(ctx context.Context, input *clou
 }
 
 func (m *fakeCWLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
-	return &m.logGroups, nil
+	output := &m.logGroups[m.logGroupsIndex]
+	m.logGroupsIndex++
+	return output, nil
 }
 
 func (m *fakeCWLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {

public/app/plugins/datasource/cloudwatch/datasource.ts

@@ -474,6 +474,13 @@ export class CloudWatchDatasource
     return logGroupNames;
   }
 
+  async describeAllLogGroups(params: DescribeLogGroupsRequest): Promise<string[]> {
+    const dataFrames = await lastValueFrom(this.makeLogActionRequest('DescribeAllLogGroups', [params]));
+
+    const logGroupNames = dataFrames[0]?.fields[0]?.values.toArray() ?? [];
+    return logGroupNames;
+  }
+
   async getLogGroupFields(params: GetLogGroupFieldsRequest): Promise<GetLogGroupFieldsResponse> {
     const dataFrames = await lastValueFrom(this.makeLogActionRequest('GetLogGroupFields', [params]));
 
@@ -663,9 +670,7 @@ export class CloudWatchDatasource
             }
           }
         }
-        // TODO: seems to be some sort of bug that we don't really send region with all queries. This means
+
-        //  if you select different than default region in editor you will get results for autocomplete from wrong
-        //  region.
         if (anyQuery.region) {
           anyQuery.region = this.replace(anyQuery.region, options.scopedVars, true, 'region');
           anyQuery.region = this.getActualRegion(anyQuery.region);

public/app/plugins/datasource/cloudwatch/types.ts

@@ -77,6 +77,7 @@ export interface CloudWatchMathExpressionQuery extends DataQuery {
 
 export type LogAction =
   | 'DescribeLogGroups'
+  | 'DescribeAllLogGroups'
   | 'GetQueryResults'
   | 'GetLogGroupFields'
   | 'GetLogEvents'

public/app/plugins/datasource/cloudwatch/variables.test.ts

@@ -19,7 +19,7 @@ ds.datasource.getRegions = jest.fn().mockResolvedValue([{ label: 'a', value: 'a'
 ds.datasource.getNamespaces = jest.fn().mockResolvedValue([{ label: 'b', value: 'b' }]);
 ds.datasource.getMetrics = jest.fn().mockResolvedValue([{ label: 'c', value: 'c' }]);
 ds.datasource.getDimensionKeys = jest.fn().mockResolvedValue([{ label: 'd', value: 'd' }]);
-ds.datasource.describeLogGroups = jest.fn().mockResolvedValue(['a', 'b']);
+ds.datasource.describeAllLogGroups = jest.fn().mockResolvedValue(['a', 'b']);
 const getDimensionValues = jest.fn().mockResolvedValue([{ label: 'e', value: 'e' }]);
 const getEbsVolumeIds = jest.fn().mockResolvedValue([{ label: 'f', value: 'f' }]);
 const getEc2InstanceAttribute = jest.fn().mockResolvedValue([{ label: 'g', value: 'g' }]);

public/app/plugins/datasource/cloudwatch/variables.ts

@@ -55,7 +55,7 @@ export class CloudWatchVariableSupport extends CustomVariableSupport<CloudWatchD
   }
 
   async handleLogGroupsQuery({ region, logGroupPrefix }: VariableQuery) {
-    const logGroups = await this.datasource.describeLogGroups({ region, logGroupNamePrefix: logGroupPrefix });
+    const logGroups = await this.datasource.describeAllLogGroups({ region, logGroupNamePrefix: logGroupPrefix });
     return logGroups.map((s) => ({
       text: s,
       value: s,