diff --git a/pkg/tsdb/cloudwatch/log_actions.go b/pkg/tsdb/cloudwatch/log_actions.go index 063bee3f095..036fd4ff885 100644 --- a/pkg/tsdb/cloudwatch/log_actions.go +++ b/pkg/tsdb/cloudwatch/log_actions.go @@ -19,8 +19,8 @@ import ( const ( limitExceededException = "LimitExceededException" - defaultLimit = int64(10) - logGroupDefaultLimit = int64(50) + defaultEventLimit = int64(10) + defaultLogGroupLimit = int64(50) ) type AWSError struct { @@ -128,6 +128,8 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, model LogQuer switch model.SubType { case "DescribeLogGroups": data, err = e.handleDescribeLogGroups(ctx, logsClient, model) + case "DescribeAllLogGroups": + data, err = e.handleDescribeAllLogGroups(ctx, logsClient, model) case "GetLogGroupFields": data, err = e.handleGetLogGroupFields(ctx, logsClient, model, query.RefID) case "StartQuery": @@ -148,7 +150,7 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, model LogQuer func (e *cloudWatchExecutor) handleGetLogEvents(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson) (*data.Frame, error) { - limit := defaultLimit + limit := defaultEventLimit if parameters.Limit != nil && *parameters.Limit > 0 { limit = *parameters.Limit } @@ -203,7 +205,7 @@ func (e *cloudWatchExecutor) handleGetLogEvents(ctx context.Context, logsClient func (e *cloudWatchExecutor) handleDescribeLogGroups(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson) (*data.Frame, error) { - logGroupLimit := logGroupDefaultLimit + logGroupLimit := defaultLogGroupLimit if parameters.Limit != nil && *parameters.Limit != 0 { logGroupLimit = *parameters.Limit } @@ -235,6 +237,40 @@ func (e *cloudWatchExecutor) handleDescribeLogGroups(ctx context.Context, return frame, nil } +func (e *cloudWatchExecutor) handleDescribeAllLogGroups(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson) (*data.Frame, error) { + var namePrefix, nextToken *string + if len(parameters.LogGroupNamePrefix) != 0 { + namePrefix = aws.String(parameters.LogGroupNamePrefix) + } + + var response *cloudwatchlogs.DescribeLogGroupsOutput + var err error + logGroupNames := []*string{} + for { + response, err = logsClient.DescribeLogGroupsWithContext(ctx, &cloudwatchlogs.DescribeLogGroupsInput{ + LogGroupNamePrefix: namePrefix, + NextToken: nextToken, + Limit: aws.Int64(defaultLogGroupLimit), + }) + if err != nil || response == nil { + return nil, err + } + + for _, logGroup := range response.LogGroups { + logGroupNames = append(logGroupNames, logGroup.LogGroupName) + } + + if response.NextToken == nil { + break + } + nextToken = response.NextToken + } + + groupNamesField := data.NewField("logGroupName", nil, logGroupNames) + frame := data.NewFrame("logGroups", groupNamesField) + return frame, nil +} + func (e *cloudWatchExecutor) executeStartQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI, parameters LogQueryJson, timeRange backend.TimeRange) (*cloudwatchlogs.StartQueryOutput, error) { startTime := timeRange.From diff --git a/pkg/tsdb/cloudwatch/log_actions_test.go b/pkg/tsdb/cloudwatch/log_actions_test.go index c6d7eb3f877..c621e4fd74b 100644 --- a/pkg/tsdb/cloudwatch/log_actions_test.go +++ b/pkg/tsdb/cloudwatch/log_actions_test.go @@ -120,8 +120,8 @@ func TestQuery_DescribeLogGroups(t *testing.T) { t.Run("Empty log group name prefix", func(t *testing.T) { cli = fakeCWLogsClient{ - logGroups: cloudwatchlogs.DescribeLogGroupsOutput{ - LogGroups: []*cloudwatchlogs.LogGroup{ + logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{ + {LogGroups: []*cloudwatchlogs.LogGroup{ { LogGroupName: aws.String("group_a"), }, @@ -131,7 +131,7 @@ func TestQuery_DescribeLogGroups(t *testing.T) { { LogGroupName: aws.String("group_c"), }, - }, + }}, }, } @@ -176,8 +176,8 @@ func TestQuery_DescribeLogGroups(t *testing.T) { t.Run("Non-empty log group name prefix", func(t *testing.T) { cli = fakeCWLogsClient{ - logGroups: cloudwatchlogs.DescribeLogGroupsOutput{ - LogGroups: []*cloudwatchlogs.LogGroup{ + logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{ + {LogGroups: []*cloudwatchlogs.LogGroup{ { LogGroupName: aws.String("group_a"), }, @@ -187,7 +187,7 @@ func TestQuery_DescribeLogGroups(t *testing.T) { { LogGroupName: aws.String("group_c"), }, - }, + }}, }, } @@ -233,6 +233,92 @@ func TestQuery_DescribeLogGroups(t *testing.T) { }) } +func TestQuery_DescribeAllLogGroups(t *testing.T) { + origNewCWLogsClient := NewCWLogsClient + t.Cleanup(func() { + NewCWLogsClient = origNewCWLogsClient + }) + + var cli fakeCWLogsClient + + NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI { + return &cli + } + + t.Run("multiple batches", func(t *testing.T) { + token := "foo" + cli = fakeCWLogsClient{ + logGroups: []cloudwatchlogs.DescribeLogGroupsOutput{ + { + LogGroups: []*cloudwatchlogs.LogGroup{ + { + LogGroupName: aws.String("group_a"), + }, + { + LogGroupName: aws.String("group_b"), + }, + { + LogGroupName: aws.String("group_c"), + }, + }, + NextToken: &token, + }, + { + LogGroups: []*cloudwatchlogs.LogGroup{ + { + LogGroupName: aws.String("group_x"), + }, + { + LogGroupName: aws.String("group_y"), + }, + { + LogGroupName: aws.String("group_z"), + }, + }, + }, + }, + } + + im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) { + return datasourceInfo{}, nil + }) + + executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures()) + resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{ + PluginContext: backend.PluginContext{ + DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{}, + }, + Queries: []backend.DataQuery{ + { + JSON: json.RawMessage(`{ + "type": "logAction", + "subtype": "DescribeAllLogGroups", + "limit": 50 + }`), + }, + }, + }) + require.NoError(t, err) + require.NotNil(t, resp) + + assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{ + "": backend.DataResponse{ + Frames: data.Frames{ + &data.Frame{ + Name: "logGroups", + Fields: []*data.Field{ + data.NewField("logGroupName", nil, []*string{ + aws.String("group_a"), aws.String("group_b"), aws.String("group_c"), aws.String("group_x"), aws.String("group_y"), aws.String("group_z"), + }), + }, + }, + }, + }, + }, + }, resp) + }) +} + func TestQuery_GetLogGroupFields(t *testing.T) { origNewCWLogsClient := NewCWLogsClient t.Cleanup(func() { diff --git a/pkg/tsdb/cloudwatch/utils_test.go b/pkg/tsdb/cloudwatch/utils_test.go index b8ea12cb8ed..97a9ccc4283 100644 --- a/pkg/tsdb/cloudwatch/utils_test.go +++ b/pkg/tsdb/cloudwatch/utils_test.go @@ -23,9 +23,11 @@ type fakeCWLogsClient struct { calls logsQueryCalls - logGroups cloudwatchlogs.DescribeLogGroupsOutput + logGroups []cloudwatchlogs.DescribeLogGroupsOutput logGroupFields cloudwatchlogs.GetLogGroupFieldsOutput queryResults cloudwatchlogs.GetQueryResultsOutput + + logGroupsIndex int } type logsQueryCalls struct { @@ -52,7 +54,9 @@ func (m *fakeCWLogsClient) StopQueryWithContext(ctx context.Context, input *clou } func (m *fakeCWLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) { - return &m.logGroups, nil + output := &m.logGroups[m.logGroupsIndex] + m.logGroupsIndex++ + return output, nil } func (m *fakeCWLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) { diff --git a/public/app/plugins/datasource/cloudwatch/datasource.ts b/public/app/plugins/datasource/cloudwatch/datasource.ts index 2281f86e735..74e204c1f18 100644 --- a/public/app/plugins/datasource/cloudwatch/datasource.ts +++ b/public/app/plugins/datasource/cloudwatch/datasource.ts @@ -474,6 +474,13 @@ export class CloudWatchDatasource return logGroupNames; } + async describeAllLogGroups(params: DescribeLogGroupsRequest): Promise { + const dataFrames = await lastValueFrom(this.makeLogActionRequest('DescribeAllLogGroups', [params])); + + const logGroupNames = dataFrames[0]?.fields[0]?.values.toArray() ?? []; + return logGroupNames; + } + async getLogGroupFields(params: GetLogGroupFieldsRequest): Promise { const dataFrames = await lastValueFrom(this.makeLogActionRequest('GetLogGroupFields', [params])); @@ -663,9 +670,7 @@ export class CloudWatchDatasource } } } - // TODO: seems to be some sort of bug that we don't really send region with all queries. This means - // if you select different than default region in editor you will get results for autocomplete from wrong - // region. + if (anyQuery.region) { anyQuery.region = this.replace(anyQuery.region, options.scopedVars, true, 'region'); anyQuery.region = this.getActualRegion(anyQuery.region); diff --git a/public/app/plugins/datasource/cloudwatch/types.ts b/public/app/plugins/datasource/cloudwatch/types.ts index efbe7917378..b68fb6dbc54 100644 --- a/public/app/plugins/datasource/cloudwatch/types.ts +++ b/public/app/plugins/datasource/cloudwatch/types.ts @@ -77,6 +77,7 @@ export interface CloudWatchMathExpressionQuery extends DataQuery { export type LogAction = | 'DescribeLogGroups' + | 'DescribeAllLogGroups' | 'GetQueryResults' | 'GetLogGroupFields' | 'GetLogEvents' diff --git a/public/app/plugins/datasource/cloudwatch/variables.test.ts b/public/app/plugins/datasource/cloudwatch/variables.test.ts index 52c70aae428..b905908477f 100644 --- a/public/app/plugins/datasource/cloudwatch/variables.test.ts +++ b/public/app/plugins/datasource/cloudwatch/variables.test.ts @@ -19,7 +19,7 @@ ds.datasource.getRegions = jest.fn().mockResolvedValue([{ label: 'a', value: 'a' ds.datasource.getNamespaces = jest.fn().mockResolvedValue([{ label: 'b', value: 'b' }]); ds.datasource.getMetrics = jest.fn().mockResolvedValue([{ label: 'c', value: 'c' }]); ds.datasource.getDimensionKeys = jest.fn().mockResolvedValue([{ label: 'd', value: 'd' }]); -ds.datasource.describeLogGroups = jest.fn().mockResolvedValue(['a', 'b']); +ds.datasource.describeAllLogGroups = jest.fn().mockResolvedValue(['a', 'b']); const getDimensionValues = jest.fn().mockResolvedValue([{ label: 'e', value: 'e' }]); const getEbsVolumeIds = jest.fn().mockResolvedValue([{ label: 'f', value: 'f' }]); const getEc2InstanceAttribute = jest.fn().mockResolvedValue([{ label: 'g', value: 'g' }]); diff --git a/public/app/plugins/datasource/cloudwatch/variables.ts b/public/app/plugins/datasource/cloudwatch/variables.ts index 78809786374..5ff8ddb3758 100644 --- a/public/app/plugins/datasource/cloudwatch/variables.ts +++ b/public/app/plugins/datasource/cloudwatch/variables.ts @@ -55,7 +55,7 @@ export class CloudWatchVariableSupport extends CustomVariableSupport ({ text: s, value: s,