IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-01-27 00:37:04 -06:00
Code Issues Packages Projects Releases Wiki Activity

Cloudwatch: Refactor log group fields request (#60909)

Browse Source 
* cloudwatch/log-group-fields-refactor

* remove not used code

* remove empty line

* fix broken test

* add tests

* Update pkg/tsdb/cloudwatch/routes/log_group_fields_test.go

Co-authored-by: Isabella Siu <Isabella.siu@grafana.com>

* pr feedback

Co-authored-by: Isabella Siu <Isabella.siu@grafana.com>
This commit is contained in:
Erik Sundell 2023-01-11 08:12:58 +01:00 committed by GitHub
parent 63ba3ccb58
commit c72ab21096
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 437 additions and 231 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.betterer.results
View File

@ -5155,8 +5155,7 @@ exports[`better eslint`] = {
"public/app/plugins/datasource/cloudwatch/language_provider.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],
[0, 0, 0, "Unexpected any. Specify a different type.", "1"],
[0, 0, 0, "Unexpected any. Specify a different type.", "2"],
[0, 0, 0, "Do not use any type assertions.", "3"]
[0, 0, 0, "Unexpected any. Specify a different type.", "2"]
],
"public/app/plugins/datasource/cloudwatch/memoizedDebounce.ts:5381": [
[0, 0, 0, "Unexpected any. Specify a different type.", "0"],

43
pkg/tsdb/cloudwatch/cloudwatch_integration_test.go
View File

@ -9,6 +9,7 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
@ -18,6 +19,7 @@ import (
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
)
@ -27,7 +29,12 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) {
origNewOAMAPI := NewOAMAPI
origNewLogsAPI := NewLogsAPI
NewOAMAPI = func(sess *session.Session) models.OAMAPIProvider { return nil }
NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider { return nil }
var logApi mocks.LogsAPI
NewLogsAPI = func(sess *session.Session) models.CloudWatchLogsAPIProvider {
return &logApi
}
t.Cleanup(func() {
NewOAMAPI = origNewOAMAPI
NewMetricsAPI = origNewMetricsAPI
@ -198,4 +205,38 @@ func Test_CloudWatch_CallResource_Integration_Test(t *testing.T) {
require.Nil(t, err)
assert.Equal(t, []resources.ResourceResponse[resources.Metric]{{Value: resources.Metric{Name: "Test_MetricName1", Namespace: "AWS/EC2"}}, {Value: resources.Metric{Name: "Test_MetricName2", Namespace: "AWS/EC2"}}, {Value: resources.Metric{Name: "Test_MetricName3", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName10", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName4", Namespace: "AWS/ECS"}}, {Value: resources.Metric{Name: "Test_MetricName5", Namespace: "AWS/Redshift"}}}, res)
})
t.Run("Should handle log group fields request", func(t *testing.T) {
logApi = mocks.LogsAPI{}
logApi.On("GetLogGroupFields", mock.Anything).Return(&cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field1"),
Percent: aws.Int64(50),
},
{
Name: aws.String("field2"),
Percent: aws.Int64(50),
},
},
}, nil)
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
req := &backend.CallResourceRequest{
Method: "GET",
Path: `/log-group-fields?region=us-east-2&logGroupName=test`,
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{ID: 0},
PluginID: "cloudwatch",
},
}
err := executor.CallResource(context.Background(), req, sender)
require.NoError(t, err)
sent := sender.Response
require.NotNil(t, sent)
require.Equal(t, http.StatusOK, sent.Status)
require.Nil(t, err)
assert.JSONEq(t, `[{"value":{"name":"field1","percent":50}},{"value":{"name":"field2","percent":50}}]`, string(sent.Body))
})
}

33
pkg/tsdb/cloudwatch/log_actions.go
View File

@ -112,8 +112,6 @@ func (e *cloudWatchExecutor) executeLogAction(ctx context.Context, logger log.Lo
var data *data.Frame = nil
switch logsQuery.SubType {
case "GetLogGroupFields":
data, err = e.handleGetLogGroupFields(ctx, logsClient, logsQuery, query.RefID)
case "StartQuery":
data, err = e.handleStartQuery(ctx, logger, logsClient, logsQuery, query.TimeRange, query.RefID)
case "StopQuery":
@ -321,37 +319,6 @@ func (e *cloudWatchExecutor) handleGetQueryResults(ctx context.Context, logsClie
return dataFrame, nil
}
func (e *cloudWatchExecutor) handleGetLogGroupFields(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI,
logsQuery models.LogsQuery, refID string) (*data.Frame, error) {
queryInput := &cloudwatchlogs.GetLogGroupFieldsInput{
LogGroupName: aws.String(logsQuery.LogGroupName),
Time: aws.Int64(logsQuery.Time),
}
getLogGroupFieldsOutput, err := logsClient.GetLogGroupFieldsWithContext(ctx, queryInput)
if err != nil {
return nil, err
}
fieldNames := make([]*string, 0)
fieldPercentages := make([]*int64, 0)
for _, logGroupField := range getLogGroupFieldsOutput.LogGroupFields {
fieldNames = append(fieldNames, logGroupField.Name)
fieldPercentages = append(fieldPercentages, logGroupField.Percent)
}
dataFrame := data.NewFrame(
refID,
data.NewField("name", nil, fieldNames),
data.NewField("percent", nil, fieldPercentages),
)
dataFrame.RefID = refID
return dataFrame, nil
}
func groupResponseFrame(frame *data.Frame, statsGroups []string) (data.Frames, error) {
var dataFrames data.Frames

77
pkg/tsdb/cloudwatch/log_actions_test.go
View File

@ -107,83 +107,6 @@ func TestQuery_GetLogEvents(t *testing.T) {
}
}
func TestQuery_GetLogGroupFields(t *testing.T) {
origNewCWLogsClient := NewCWLogsClient
t.Cleanup(func() {
NewCWLogsClient = origNewCWLogsClient
})
var cli fakeCWLogsClient
NewCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return &cli
}
cli = fakeCWLogsClient{
logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
},
}
const refID = "A"
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return DataSource{Settings: models.CloudWatchSettings{}}, nil
})
executor := newExecutor(im, newTestConfig(), &fakeSessionCache{}, featuremgmt.WithFeatures())
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
RefID: refID,
JSON: json.RawMessage(`{
"type": "logAction",
"subtype": "GetLogGroupFields",
"logGroupName": "group_a",
"limit": 50
}`),
},
},
})
require.NoError(t, err)
require.NotNil(t, resp)
expFrame := &data.Frame{
Name: refID,
Fields: []*data.Field{
data.NewField("name", nil, []*string{
aws.String("field_a"), aws.String("field_b"), aws.String("field_c"),
}),
data.NewField("percent", nil, []*int64{
aws.Int64(100), aws.Int64(30), aws.Int64(55),
}),
},
}
expFrame.RefID = refID
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
refID: backend.DataResponse{
Frames: data.Frames{expFrame},
},
},
}, resp)
}
func TestQuery_StartQuery(t *testing.T) {
origNewCWLogsClient := NewCWLogsClient
t.Cleanup(func() {

12
pkg/tsdb/cloudwatch/mocks/logs.go
View File

@ -16,6 +16,12 @@ func (l *LogsAPI) DescribeLogGroups(input *cloudwatchlogs.DescribeLogGroupsInput
return args.Get(0).(*cloudwatchlogs.DescribeLogGroupsOutput), args.Error(1)
}
func (l *LogsAPI) GetLogGroupFields(input *cloudwatchlogs.GetLogGroupFieldsInput) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {
args := l.Called(input)
return args.Get(0).(*cloudwatchlogs.GetLogGroupFieldsOutput), args.Error(1)
}
type LogsService struct {
mock.Mock
}
@ -26,6 +32,12 @@ func (l *LogsService) GetLogGroups(request resources.LogGroupsRequest) ([]resour
return args.Get(0).([]resources.ResourceResponse[resources.LogGroup]), args.Error(1)
}
func (l *LogsService) GetLogGroupFields(request resources.LogGroupFieldsRequest) ([]resources.ResourceResponse[resources.LogGroupField], error) {
args := l.Called(request)
return args.Get(0).([]resources.ResourceResponse[resources.LogGroupField]), args.Error(1)
}
type MockFeatures struct {
mock.Mock
}

2
pkg/tsdb/cloudwatch/models/api.go
View File

@ -32,6 +32,7 @@ type ListMetricsProvider interface {
type LogGroupsProvider interface {
GetLogGroups(request resources.LogGroupsRequest) ([]resources.ResourceResponse[resources.LogGroup], error)
GetLogGroupFields(request resources.LogGroupFieldsRequest) ([]resources.ResourceResponse[resources.LogGroupField], error)
}
type AccountsProvider interface {
@ -50,6 +51,7 @@ type CloudWatchMetricsAPIProvider interface {
type CloudWatchLogsAPIProvider interface {
DescribeLogGroups(*cloudwatchlogs.DescribeLogGroupsInput) (*cloudwatchlogs.DescribeLogGroupsOutput, error)
GetLogGroupFields(*cloudwatchlogs.GetLogGroupFieldsInput) (*cloudwatchlogs.GetLogGroupFieldsOutput, error)
}
type OAMAPIProvider interface {

31
pkg/tsdb/cloudwatch/models/resources/log_group_fields_resource_request.go Normal file
View File

@ -0,0 +1,31 @@
package resources
import (
"fmt"
"net/url"
)
type LogGroupFieldsRequest struct {
ResourceRequest
LogGroupName string
LogGroupARN string
}
func ParseLogGroupFieldsRequest(parameters url.Values) (LogGroupFieldsRequest, error) {
resourceRequest, err := getResourceRequest(parameters)
if err != nil {
return LogGroupFieldsRequest{}, err
}
request := LogGroupFieldsRequest{
ResourceRequest: *resourceRequest,
LogGroupName: parameters.Get("logGroupName"),
LogGroupARN: parameters.Get("logGroupArn"),
}
if request.LogGroupName == "" && request.LogGroupARN == "" {
return LogGroupFieldsRequest{}, fmt.Errorf("you need to specify either logGroupName or logGroupArn")
}
return request, nil
}

32
pkg/tsdb/cloudwatch/models/resources/log_group_fields_resource_request_test.go Normal file
View File

@ -0,0 +1,32 @@
package resources
import (
"fmt"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestLogGroupFieldsRequest(t *testing.T) {
t.Run("Should parse valid parameters", func(t *testing.T) {
request, err := ParseLogGroupFieldsRequest(map[string][]string{
"region": {"us-east-1"},
"logGroupName": {"my-log-group"},
"logGroupArn": {"arn:aws:logs:us-east-1:123456789012:log-group:my-log-group"}},
)
require.NoError(t, err)
assert.Equal(t, "us-east-1", request.Region)
assert.Equal(t, "my-log-group", request.LogGroupName)
assert.Equal(t, "arn:aws:logs:us-east-1:123456789012:log-group:my-log-group", request.LogGroupARN)
})
t.Run("Should return an error if arn and name is missing ", func(t *testing.T) {
request, err := ParseLogGroupFieldsRequest(map[string][]string{
"region": {"us-east-1"},
},
)
require.Empty(t, request)
require.Error(t, fmt.Errorf("you need to specify either logGroupName or logGroupArn"), err)
})
}

5
pkg/tsdb/cloudwatch/models/resources/types.go
View File

@ -33,3 +33,8 @@ type LogGroup struct {
Arn string `json:"arn"`
Name string `json:"name"`
}
type LogGroupField struct {
Percent int64 `json:"percent"`
Name string `json:"name"`
}

1
pkg/tsdb/cloudwatch/resource_handler.go
View File

@ -25,6 +25,7 @@ func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux {
mux.HandleFunc("/dimension-keys", routes.ResourceRequestMiddleware(routes.DimensionKeysHandler, logger, e.getRequestContext))
mux.HandleFunc("/accounts", routes.ResourceRequestMiddleware(routes.AccountsHandler, logger, e.getRequestContext))
mux.HandleFunc("/namespaces", routes.ResourceRequestMiddleware(routes.NamespacesHandler, logger, e.getRequestContext))
mux.HandleFunc("/log-group-fields", routes.ResourceRequestMiddleware(routes.LogGroupFieldsHandler, logger, e.getRequestContext))
return mux
}

35
pkg/tsdb/cloudwatch/routes/log_group_fields.go Normal file
View File

@ -0,0 +1,35 @@
package routes
import (
"encoding/json"
"net/http"
"net/url"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources"
)
func LogGroupFieldsHandler(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, parameters url.Values) ([]byte, *models.HttpError) {
request, err := resources.ParseLogGroupFieldsRequest(parameters)
if err != nil {
return nil, models.NewHttpError("error in LogGroupFieldsHandler", http.StatusBadRequest, err)
}
service, err := newLogGroupsService(pluginCtx, reqCtxFactory, request.Region)
if err != nil {
return nil, models.NewHttpError("newLogGroupsService error", http.StatusInternalServerError, err)
}
logGroupFields, err := service.GetLogGroupFields(request)
if err != nil {
return nil, models.NewHttpError("GetLogGroupFields error", http.StatusInternalServerError, err)
}
logGroupsResponse, err := json.Marshal(logGroupFields)
if err != nil {
return nil, models.NewHttpError("LogGroupFieldsHandler json error", http.StatusInternalServerError, err)
}
return logGroupsResponse, nil
}

77
pkg/tsdb/cloudwatch/routes/log_group_fields_test.go Normal file
View File

@ -0,0 +1,77 @@
package routes
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/mocks"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models"
"github.com/grafana/grafana/pkg/tsdb/cloudwatch/models/resources"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
func TestLogGroupFieldsRoute(t *testing.T) {
mockFeatures := mocks.MockFeatures{}
reqCtxFunc := func(pluginCtx backend.PluginContext, region string) (reqCtx models.RequestContext, err error) {
return models.RequestContext{Features: &mockFeatures}, err
}
t.Run("returns 400 if an invalid LogGroupFieldsRequest is used", func(t *testing.T) {
rr := httptest.NewRecorder()
req := httptest.NewRequest("GET", `/log-group-fields?region=us-east-2`, nil)
handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupFieldsHandler, logger, nil))
handler.ServeHTTP(rr, req)
assert.Equal(t, http.StatusBadRequest, rr.Code)
assert.Equal(t, `{"Message":"error in LogGroupFieldsHandler: you need to specify either logGroupName or logGroupArn","Error":"you need to specify either logGroupName or logGroupArn","StatusCode":400}`, rr.Body.String())
})
t.Run("returns 500 if GetLogGroupFields method fails", func(t *testing.T) {
mockLogsService := mocks.LogsService{}
mockLogsService.On("GetLogGroupFields", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroupField]{}, fmt.Errorf("error from api"))
newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) {
return &mockLogsService, nil
}
rr := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/log-group-fields?region=us-east-2&logGroupName=test", nil)
handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupFieldsHandler, logger, reqCtxFunc))
handler.ServeHTTP(rr, req)
assert.Equal(t, http.StatusInternalServerError, rr.Code)
assert.Equal(t, `{"Message":"GetLogGroupFields error: error from api","Error":"error from api","StatusCode":500}`, rr.Body.String())
})
t.Run("returns valid json response if everything is ok", func(t *testing.T) {
mockLogsService := mocks.LogsService{}
mockLogsService.On("GetLogGroupFields", mock.Anything).Return([]resources.ResourceResponse[resources.LogGroupField]{
{
AccountId: new(string),
Value: resources.LogGroupField{
Name: "field1",
Percent: 50,
},
},
{
AccountId: new(string),
Value: resources.LogGroupField{
Name: "field2",
Percent: 50,
},
},
}, nil)
newLogGroupsService = func(pluginCtx backend.PluginContext, reqCtxFactory models.RequestContextFactoryFunc, region string) (models.LogGroupsProvider, error) {
return &mockLogsService, nil
}
rr := httptest.NewRecorder()
req := httptest.NewRequest("GET", "/log-group-fields?region=us-east-2&logGroupName=test", nil)
handler := http.HandlerFunc(ResourceRequestMiddleware(LogGroupFieldsHandler, logger, reqCtxFunc))
handler.ServeHTTP(rr, req)
assert.Equal(t, http.StatusOK, rr.Code)
assert.JSONEq(t, `[{"accountId":"","value":{"name":"field1","percent":50}},{"accountId":"","value":{"name":"field2","percent":50}}]`, rr.Body.String())
})
}

2
pkg/tsdb/cloudwatch/routes/log_groups_test.go
View File

@ -16,7 +16,7 @@ import (
"github.com/stretchr/testify/mock"
)
func Test_log_groups_route(t *testing.T) {
func TestLogGroupsRoute(t *testing.T) {
origLogGroupsService := newLogGroupsService
t.Cleanup(func() {
newLogGroupsService = origLogGroupsService

28
pkg/tsdb/cloudwatch/services/logs.go → pkg/tsdb/cloudwatch/services/log_groups.go
View File

@ -51,3 +51,31 @@ func (s *LogGroupsService) GetLogGroups(req resources.LogGroupsRequest) ([]resou
return result, nil
}
func (s *LogGroupsService) GetLogGroupFields(request resources.LogGroupFieldsRequest) ([]resources.ResourceResponse[resources.LogGroupField], error) {
input := &cloudwatchlogs.GetLogGroupFieldsInput{
LogGroupName: aws.String(request.LogGroupName),
}
// we should use LogGroupIdentifier instead of LogGroupName, but currently the api doesn't accept LogGroupIdentifier. need to check if it's a bug or not.
// if request.LogGroupARN != "" {
// input.LogGroupIdentifier = aws.String(strings.TrimSuffix(request.LogGroupARN, ":*"))
// input.LogGroupName = nil
// }
getLogGroupFieldsOutput, err := s.logGroupsAPI.GetLogGroupFields(input)
if err != nil {
return nil, err
}
result := make([]resources.ResourceResponse[resources.LogGroupField], 0)
for _, logGroupField := range getLogGroupFieldsOutput.LogGroupFields {
result = append(result, resources.ResourceResponse[resources.LogGroupField]{
Value: resources.LogGroupField{
Name: *logGroupField.Name,
Percent: *logGroupField.Percent,
},
})
}
return result, nil
}

109
pkg/tsdb/cloudwatch/services/logs_test.go → pkg/tsdb/cloudwatch/services/log_groups_test.go
View File

@ -12,7 +12,7 @@ import (
"github.com/stretchr/testify/mock"
)
func Test_GetLogGroups(t *testing.T) {
func TestGetLogGroups(t *testing.T) {
t.Run("Should map log groups response", func(t *testing.T) {
mockLogsAPI := &mocks.LogsAPI{}
mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(
@ -88,7 +88,7 @@ func Test_GetLogGroups(t *testing.T) {
})
}
func Test_GetLogGroups_crossAccountQuerying(t *testing.T) {
func TestGetLogGroupsCrossAccountQuerying(t *testing.T) {
t.Run("Should not includeLinkedAccounts or accountId if isCrossAccountEnabled is set to false", func(t *testing.T) {
mockLogsAPI := &mocks.LogsAPI{}
mockLogsAPI.On("DescribeLogGroups", mock.Anything).Return(&cloudwatchlogs.DescribeLogGroupsOutput{}, nil)
@ -198,3 +198,108 @@ func Test_GetLogGroups_crossAccountQuerying(t *testing.T) {
})
})
}
func TestGetLogGroupFields(t *testing.T) {
t.Run("Should map log group fields response", func(t *testing.T) {
mockLogsAPI := &mocks.LogsAPI{}
mockLogsAPI.On("GetLogGroupFields", mock.Anything).Return(
&cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: utils.Pointer("field1"),
Percent: utils.Pointer(int64(10)),
}, {
Name: utils.Pointer("field2"),
Percent: utils.Pointer(int64(10)),
}, {
Name: utils.Pointer("field3"),
Percent: utils.Pointer(int64(10)),
},
},
}, nil)
service := NewLogGroupsService(mockLogsAPI, false)
resp, err := service.GetLogGroupFields(resources.LogGroupFieldsRequest{})
assert.NoError(t, err)
assert.Equal(t, []resources.ResourceResponse[resources.LogGroupField]{
{
Value: resources.LogGroupField{
Name: "field1",
Percent: 10,
},
},
{
Value: resources.LogGroupField{
Name: "field2",
Percent: 10,
},
},
{
Value: resources.LogGroupField{
Name: "field3",
Percent: 10,
},
},
}, resp)
})
// uncomment this test if when it's possible to pass only LogGroupIdentifier to the api
// t.Run("Should only set LogGroupIdentifier as api input in case both LogGroupName and LogGroupARN are specified", func(t *testing.T) {
// mockLogsAPI := &mocks.LogsAPI{}
// mockLogsAPI.On("GetLogGroupFields", mock.Anything).Return(
// &cloudwatchlogs.GetLogGroupFieldsOutput{}, nil)
// service := NewLogGroupsService(mockLogsAPI, false)
// resp, err := service.GetLogGroupFields(resources.LogGroupFieldsRequest{
// LogGroupName: "logGroupName",
// LogGroupARN: "logGroupARN",
// })
// mockLogsAPI.AssertCalled(t, "GetLogGroupFields", &cloudwatchlogs.GetLogGroupFieldsInput{
// LogGroupIdentifier: utils.Pointer("logGroupARN"),
// LogGroupName: nil,
// })
// assert.NotNil(t, resp)
// assert.NoError(t, err)
// })
// remove this test once the above test is uncommented
t.Run("Should only set LogGroupName as api input in case both LogGroupName and LogGroupARN are specified", func(t *testing.T) {
mockLogsAPI := &mocks.LogsAPI{}
mockLogsAPI.On("GetLogGroupFields", mock.Anything).Return(
&cloudwatchlogs.GetLogGroupFieldsOutput{}, nil)
service := NewLogGroupsService(mockLogsAPI, false)
resp, err := service.GetLogGroupFields(resources.LogGroupFieldsRequest{
LogGroupName: "logGroupName",
LogGroupARN: "logGroupARN",
})
mockLogsAPI.AssertCalled(t, "GetLogGroupFields", &cloudwatchlogs.GetLogGroupFieldsInput{
LogGroupIdentifier: nil,
LogGroupName: utils.Pointer("logGroupName"),
})
assert.NotNil(t, resp)
assert.NoError(t, err)
})
t.Run("Should only set LogGroupName as api input in case only LogGroupName is specified", func(t *testing.T) {
mockLogsAPI := &mocks.LogsAPI{}
mockLogsAPI.On("GetLogGroupFields", mock.Anything).Return(
&cloudwatchlogs.GetLogGroupFieldsOutput{}, nil)
service := NewLogGroupsService(mockLogsAPI, false)
resp, err := service.GetLogGroupFields(resources.LogGroupFieldsRequest{
LogGroupName: "logGroupName",
LogGroupARN: "",
})
mockLogsAPI.AssertCalled(t, "GetLogGroupFields", &cloudwatchlogs.GetLogGroupFieldsInput{
LogGroupIdentifier: nil,
LogGroupName: utils.Pointer("logGroupName"),
})
assert.NotNil(t, resp)
assert.NoError(t, err)
})
}

4
pkg/tsdb/cloudwatch/test_utils.go
View File

@ -184,6 +184,10 @@ func (c fakeCheckHealthClient) DescribeLogGroups(input *cloudwatchlogs.DescribeL
return nil, nil
}
func (c fakeCheckHealthClient) GetLogGroupFields(input *cloudwatchlogs.GetLogGroupFieldsInput) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {
return nil, nil
}
func newTestConfig() *setting.Cfg {
return &setting.Cfg{AWSAllowedAuthProviders: []string{"default"}, AWSAssumeRoleEnabled: true, AWSListMetricsPageLimit: 1000}
}

14
public/app/plugins/datasource/cloudwatch/api.ts
View File

@ -17,6 +17,8 @@ import {
Account,
ResourceRequest,
ResourceResponse,
GetLogGroupFieldsRequest,
LogGroupField,
} from './types';
export interface SelectableResourceValue extends SelectableValue<string> {
@ -70,6 +72,18 @@ export class CloudWatchAPI extends CloudWatchRequest {
});
}
async getLogGroupFields({
region,
arn,
logGroupName,
}: GetLogGroupFieldsRequest): Promise<Array<ResourceResponse<LogGroupField>>> {
return this.memoizedGetRequest<Array<ResourceResponse<LogGroupField>>>('log-group-fields', {
region: this.templateSrv.replace(this.getActualRegion(region)),
logGroupName: this.templateSrv.replace(logGroupName, {}),
logGroupArn: this.templateSrv.replace(arn),
});
}
async describeAllLogGroups(params: DescribeLogGroupsRequest) {
return this.memoizedGetRequest<SelectableResourceValue[]>('all-log-groups', {
...params,

5
public/app/plugins/datasource/cloudwatch/components/LogsQueryField.tsx
View File

@ -61,7 +61,8 @@ export const CloudWatchLogsQueryField = (props: CloudWatchLogsQueryFieldProps) =
};
const onTypeahead = async (typeahead: TypeaheadInput): Promise<TypeaheadOutput> => {
const { logGroupNames } = query;
const { datasource, query } = props;
const { logGroups } = query;
if (!datasource.languageProvider) {
return { suggestions: [] };
@ -76,7 +77,7 @@ export const CloudWatchLogsQueryField = (props: CloudWatchLogsQueryFieldProps) =
{
history,
absoluteRange,
logGroupNames,
logGroups: logGroups,
region: query.region,
}
);

10
public/app/plugins/datasource/cloudwatch/language_provider.test.ts
View File

@ -15,7 +15,7 @@ import {
STRING_FUNCTIONS,
FIELD_AND_FILTER_FUNCTIONS,
} from './syntax';
import { GetLogGroupFieldsResponse } from './types';
import { LogGroupField, ResourceResponse } from './types';
const fields = ['field1', '@message'];
@ -109,9 +109,9 @@ async function runSuggestionTest(query: string, expectedItems: string[][]) {
function makeDatasource(): CloudWatchDatasource {
return {
logsQueryRunner: {
getLogGroupFields(): Promise<GetLogGroupFieldsResponse> {
return Promise.resolve({ logGroupFields: [{ name: 'field1' }, { name: '@message' }] });
api: {
getLogGroupFields(): Promise<Array<ResourceResponse<LogGroupField>>> {
return Promise.resolve([{ value: { name: 'field1' } }, { value: { name: '@message' } }]);
},
},
} as any;
@ -131,7 +131,7 @@ function getProvideCompletionItems(query: string): Promise<TypeaheadOutput> {
{
value,
} as any,
{ logGroupNames: ['logGroup1'], region: 'custom' }
{ logGroups: [{ name: 'logGroup1', arn: 'logGroup1' }], region: 'custom' }
);
}

70
public/app/plugins/datasource/cloudwatch/language_provider.ts
View File

@ -1,4 +1,3 @@
import { sortedUniq } from 'lodash';
import Prism, { Grammar } from 'prismjs';
import { lastValueFrom } from 'rxjs';
@ -16,14 +15,14 @@ import syntax, {
QUERY_COMMANDS,
STRING_FUNCTIONS,
} from './syntax';
import { CloudWatchQuery, TSDBResponse } from './types';
import { CloudWatchQuery, LogGroup, TSDBResponse } from './types';
export type CloudWatchHistoryItem = HistoryItem<CloudWatchQuery>;
type TypeaheadContext = {
history?: CloudWatchHistoryItem[];
absoluteRange?: AbsoluteTimeRange;
logGroupNames?: string[];
logGroups?: LogGroup[];
region: string;
};
@ -106,7 +105,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider {
}
if (isInsideFunctionParenthesis(curToken)) {
return await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default');
return await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default');
}
if (isAfterKeyword('by', curToken)) {
@ -127,44 +126,20 @@ export class CloudWatchLanguageProvider extends LanguageProvider {
};
}
private fetchedFieldsCache:
| {
time: number;
logGroups: string[];
fields: string[];
}
| undefined;
private fetchFields = async (logGroups: string[], region: string): Promise<string[]> => {
if (
this.fetchedFieldsCache &&
Date.now() - this.fetchedFieldsCache.time < 30 * 1000 &&
sortedUniq(this.fetchedFieldsCache.logGroups).join('|') === sortedUniq(logGroups).join('|')
) {
return this.fetchedFieldsCache.fields;
}
private fetchFields = async (logGroups: LogGroup[], region: string): Promise<string[]> => {
const results = await Promise.all(
logGroups.map((logGroup) => this.datasource.logsQueryRunner.getLogGroupFields({ logGroupName: logGroup, region }))
logGroups.map((logGroup) =>
this.datasource.api
.getLogGroupFields({ logGroupName: logGroup.name, arn: logGroup.arn, region })
.then((fields) => fields.filter((f) => f).map((f) => f.value.name ?? ''))
)
);
const fields = [
...new Set<string>(
results.reduce((acc: string[], cur) => acc.concat(cur.logGroupFields?.map((f) => f.name) as string[]), [])
).values(),
];
this.fetchedFieldsCache = {
time: Date.now(),
logGroups,
fields,
};
return fields;
return results.flat();
};
private handleKeyword = async (context?: TypeaheadContext): Promise<TypeaheadOutput> => {
const suggs = await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default');
const suggs = await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default');
const functionSuggestions: CompletionItemGroup[] = [
{
searchFunctionType: SearchFunctionType.Prefix,
@ -192,7 +167,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider {
if (queryCommand === 'parse') {
if (currentTokenIsFirstArg) {
return await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default');
return await this.getFieldCompletionItems(context?.logGroups ?? [], context?.region || 'default');
}
}
@ -210,7 +185,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider {
if (['display', 'fields'].includes(queryCommand)) {
const typeaheadOutput = await this.getFieldCompletionItems(
context?.logGroupNames ?? [],
context?.logGroups ?? [],
context?.region || 'default'
);
typeaheadOutput.suggestions.push(...this.getFieldAndFilterFunctionCompletionItems().suggestions);
@ -229,7 +204,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider {
}
if (queryCommand === 'filter' && currentTokenIsFirstArg) {
const sugg = await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default');
const sugg = await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default');
const boolFuncs = this.getBoolFuncCompletionItems();
sugg.suggestions.push(...boolFuncs.suggestions);
return sugg;
@ -243,7 +218,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider {
context?: TypeaheadContext
): Promise<TypeaheadOutput> {
if (isFirstArgument) {
return await this.getFieldCompletionItems(context?.logGroupNames ?? [], context?.region || 'default');
return await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default');
} else if (isTokenType(prevNonWhitespaceToken(curToken), 'field-name')) {
// suggest sort options
return {
@ -266,10 +241,7 @@ export class CloudWatchLanguageProvider extends LanguageProvider {
}
private handleComparison = async (context?: TypeaheadContext) => {
const fieldsSuggestions = await this.getFieldCompletionItems(
context?.logGroupNames ?? [],
context?.region || 'default'
);
const fieldsSuggestions = await this.getFieldCompletionItems(context?.logGroups, context?.region || 'default');
const comparisonSuggestions = this.getComparisonCompletionItems();
fieldsSuggestions.suggestions.push(...comparisonSuggestions.suggestions);
return fieldsSuggestions;
@ -321,9 +293,15 @@ export class CloudWatchLanguageProvider extends LanguageProvider {
};
};
private getFieldCompletionItems = async (logGroups: string[], region: string): Promise<TypeaheadOutput> => {
const fields = await this.fetchFields(logGroups, region);
private getFieldCompletionItems = async (
logGroups: LogGroup[] | undefined,
region: string
): Promise<TypeaheadOutput> => {
if (!logGroups) {
return { suggestions: [] };
}
const fields = await this.fetchFields(logGroups, region);
return {
suggestions: [
{

30
public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.test.ts
View File

@ -1,6 +1,6 @@
import { interval, lastValueFrom, of } from 'rxjs';
import { dataFrameToJSON, DataQueryErrorType, FieldType, LogLevel, LogRowModel, MutableDataFrame } from '@grafana/data';
import { DataQueryErrorType, FieldType, LogLevel, LogRowModel, MutableDataFrame } from '@grafana/data';
import { genMockFrames, setupMockedLogsQueryRunner } from '../__mocks__/LogsQueryRunner';
import { validLogsQuery } from '../__mocks__/queries';
@ -51,34 +51,6 @@ describe('CloudWatchLogsQueryRunner', () => {
});
});
describe('getLogGroupFields', () => {
it('passes region correctly', async () => {
const { runner, fetchMock } = setupMockedLogsQueryRunner();
fetchMock.mockReturnValueOnce(
of({
data: {
results: {
A: {
frames: [
dataFrameToJSON(
new MutableDataFrame({
fields: [
{ name: 'key', values: [] },
{ name: 'val', values: [] },
],
})
),
],
},
},
},
})
);
await runner.getLogGroupFields({ region: 'us-west-1', logGroupName: 'test' });
expect(fetchMock.mock.calls[0][0].data.queries[0].region).toBe('us-west-1');
});
});
describe('logs query', () => {
beforeEach(() => {
jest.spyOn(rxjsUtils, 'increasingInterval').mockImplementation(() => interval(100));

13
public/app/plugins/datasource/cloudwatch/query-runner/CloudWatchLogsQueryRunner.ts
View File

@ -42,7 +42,6 @@ import {
DescribeLogGroupsRequest,
GetLogEventsRequest,
GetLogGroupFieldsRequest,
GetLogGroupFieldsResponse,
LogAction,
StartQueryRequest,
} from '../types';
@ -415,18 +414,6 @@ export class CloudWatchLogsQueryRunner extends CloudWatchRequest {
};
};
async getLogGroupFields(params: GetLogGroupFieldsRequest): Promise<GetLogGroupFieldsResponse> {
const dataFrames = await lastValueFrom(this.makeLogActionRequest('GetLogGroupFields', [params]));
const fieldNames = dataFrames[0].fields[0].values.toArray();
const fieldPercentages = dataFrames[0].fields[1].values.toArray();
const getLogGroupFieldsResponse = {
logGroupFields: fieldNames.map((val, i) => ({ name: val, percent: fieldPercentages[i] })) ?? [],
};
return getLogGroupFieldsResponse;
}
private filterQuery(query: CloudWatchLogsQuery) {
const hasMissingLegacyLogGroupNames = !query.logGroupNames?.length;
const hasMissingLogGroups = !query.logGroups?.length;

32
public/app/plugins/datasource/cloudwatch/types.ts
View File

@ -267,36 +267,17 @@ export interface TSDBTimeSeries {
}
export type TSDBTimePoint = [number, number];
export interface GetLogGroupFieldsRequest {
/**
* The name of the log group to search.
*/
logGroupName: string;
/**
* The time to set as the center of the query. If you specify time, the 8 minutes before and 8 minutes after this time are searched. If you omit time, the past 15 minutes are queried. The time value is specified as epoch time, the number of seconds since January 1, 1970, 00:00:00 UTC.
*/
time?: number;
region: string;
}
export interface LogGroupField {
/**
* The name of a log field.
*/
name?: string;
name: string;
/**
* The percentage of log events queried that contained the field.
*/
percent?: number;
}
export interface GetLogGroupFieldsResponse {
/**
* The array of fields found in the query. Each object in the array contains the name of the field, along with the percentage of time it appeared in the log events that were queried.
*/
logGroupFields?: LogGroupField[];
}
export interface StartQueryRequest {
/**
* The log group on which to perform the query. A StartQuery operation must include a logGroupNames or a logGroupName parameter, but not both.
@ -423,6 +404,17 @@ export interface ResourceRequest {
accountId?: string;
}
export interface GetLogGroupFieldsRequest extends ResourceRequest {
/**
* The log group identifier
*/
arn?: string;
/**
* The name of the log group to search.
*/
logGroupName: string;
}
export interface GetDimensionKeysRequest extends ResourceRequest {
metricName?: string;
namespace?: string;