IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

cloudwatch: Consolidate client logic (#25555)

Browse Source 
* cloudwatch: Consolidate client logic

Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com>
This commit is contained in:
Patrik Karlström
2020-07-23 18:52:22 +02:00
committed by GitHub
parent 80edbbe314
commit 43ef052d57
9 changed files with 1147 additions and 630 deletions
Download Patch File Download Diff File Expand all files Collapse all files

152
pkg/tsdb/cloudwatch/cloudwatch.go
View File

@@ -9,11 +9,17 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
@@ -45,75 +51,95 @@ var plog = log.New("tsdb.cloudwatch")
var aliasFormat = regexp.MustCompile(`\{\{\s*(.+?)\s*\}\}`)
func init() {
tsdb.RegisterTsdbQueryEndpoint("cloudwatch", newcloudWatchExecutor)
tsdb.RegisterTsdbQueryEndpoint("cloudwatch", func(ds *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
return newExecutor(), nil
})
}
func newcloudWatchExecutor(datasource *models.DataSource) (tsdb.TsdbQueryEndpoint, error) {
e := &cloudWatchExecutor{
DataSource: datasource,
func newExecutor() *cloudWatchExecutor {
return &cloudWatchExecutor{
logsClientsByRegion: map[string]cloudwatchlogsiface.CloudWatchLogsAPI{},
}
dsInfo := e.getDSInfo(defaultRegion)
defaultLogsClient, err := retrieveLogsClient(dsInfo)
if err != nil {
return nil, err
}
e.logsClientsByRegion = map[string]*cloudwatchlogs.CloudWatchLogs{
dsInfo.Region: defaultLogsClient,
defaultRegion: defaultLogsClient,
}
return e, nil
}
// cloudWatchExecutor executes CloudWatch requests.
type cloudWatchExecutor struct {
*models.DataSource
ec2Svc ec2iface.EC2API
rgtaSvc resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
logsClientsByRegion map[string](*cloudwatchlogs.CloudWatchLogs)
mux sync.Mutex
ec2Client ec2iface.EC2API
rgtaClient resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
logsClientsByRegion map[string]cloudwatchlogsiface.CloudWatchLogsAPI
mtx sync.Mutex
}
func (e *cloudWatchExecutor) getCWClient(region string) (*cloudwatch.CloudWatch, error) {
datasourceInfo := e.getDSInfo(region)
cfg, err := getAwsConfig(datasourceInfo)
func (e *cloudWatchExecutor) newSession(region string) (*session.Session, error) {
dsInfo := e.getDSInfo(region)
creds, err := getCredentials(dsInfo)
if err != nil {
return nil, err
}
sess, err := newSession(cfg)
cfg := &aws.Config{
Region: aws.String(dsInfo.Region),
Credentials: creds,
}
return newSession(cfg)
}
func (e *cloudWatchExecutor) getCWClient(region string) (cloudwatchiface.CloudWatchAPI, error) {
sess, err := e.newSession(region)
if err != nil {
return nil, err
}
client := cloudwatch.New(sess, cfg)
client.Handlers.Send.PushFront(func(r *request.Request) {
r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
})
return client, nil
return newCWClient(sess), nil
}
func (e *cloudWatchExecutor) getCWLogsClient(region string) (*cloudwatchlogs.CloudWatchLogs, error) {
e.mux.Lock()
defer e.mux.Unlock()
func (e *cloudWatchExecutor) getCWLogsClient(region string) (cloudwatchlogsiface.CloudWatchLogsAPI, error) {
e.mtx.Lock()
defer e.mtx.Unlock()
if logsClient, ok := e.logsClientsByRegion[region]; ok {
return logsClient, nil
}
dsInfo := e.getDSInfo(region)
newLogsClient, err := retrieveLogsClient(dsInfo)
sess, err := e.newSession(region)
if err != nil {
return nil, err
}
e.logsClientsByRegion[region] = newLogsClient
logsClient := newCWLogsClient(sess)
e.logsClientsByRegion[region] = logsClient
return newLogsClient, nil
return logsClient, nil
}
func (e *cloudWatchExecutor) getEC2Client(region string) (ec2iface.EC2API, error) {
if e.ec2Client != nil {
return e.ec2Client, nil
}
sess, err := e.newSession(region)
if err != nil {
return nil, err
}
e.ec2Client = newEC2Client(sess)
return e.ec2Client, nil
}
func (e *cloudWatchExecutor) getRGTAClient(region string) (resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI,
error) {
if e.rgtaClient != nil {
return e.rgtaClient, nil
}
sess, err := e.newSession(region)
if err != nil {
return nil, err
}
e.rgtaClient = newRGTAClient(sess)
return e.rgtaClient, nil
}
func (e *cloudWatchExecutor) alertQuery(ctx context.Context, logsClient cloudwatchlogsiface.CloudWatchLogsAPI,
@@ -279,26 +305,44 @@ func (e *cloudWatchExecutor) getDSInfo(region string) *datasourceInfo {
}
}
func retrieveLogsClient(dsInfo *datasourceInfo) (*cloudwatchlogs.CloudWatchLogs, error) {
cfg, err := getAwsConfig(dsInfo)
if err != nil {
return nil, err
}
sess, err := newSession(cfg)
if err != nil {
return nil, err
}
client := cloudwatchlogs.New(sess, cfg)
func isTerminated(queryStatus string) bool {
return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout"
}
// CloudWatch client factory.
//
// Stubbable by tests.
var newCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
client := cloudwatch.New(sess)
client.Handlers.Send.PushFront(func(r *request.Request) {
r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
})
return client, nil
return client
}
func isTerminated(queryStatus string) bool {
return queryStatus == "Complete" || queryStatus == "Cancelled" || queryStatus == "Failed" || queryStatus == "Timeout"
// CloudWatch logs client factory.
//
// Stubbable by tests.
var newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
client := cloudwatchlogs.New(sess)
client.Handlers.Send.PushFront(func(r *request.Request) {
r.HTTPRequest.Header.Set("User-Agent", fmt.Sprintf("Grafana/%s", setting.BuildVersion))
})
return client
}
// EC2 client factory.
//
// Stubbable by tests.
var newEC2Client = func(provider client.ConfigProvider) ec2iface.EC2API {
return ec2.New(provider)
}
// RGTA client factory.
//
// Stubbable by tests.
var newRGTAClient = func(provider client.ConfigProvider) resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI {
return resourcegroupstaggingapi.New(provider)
}

14
pkg/tsdb/cloudwatch/credentials.go
View File

@@ -174,17 +174,3 @@ func ecsCredProvider(sess *session.Session, uri string) credentials.Provider {
func ec2RoleProvider(sess client.ConfigProvider) credentials.Provider {
return &ec2rolecreds.EC2RoleProvider{Client: newEC2Metadata(sess), ExpiryWindow: 5 * time.Minute}
}
func getAwsConfig(dsInfo *datasourceInfo) (*aws.Config, error) {
creds, err := getCredentials(dsInfo)
if err != nil {
return nil, err
}
cfg := &aws.Config{
Region: aws.String(dsInfo.Region),
Credentials: creds,
}
return cfg, nil
}

582
pkg/tsdb/cloudwatch/log_actions_test.go
View File

@@ -7,7 +7,9 @@ import (
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/tsdb"
@@ -16,84 +18,351 @@ import (
"github.com/stretchr/testify/require"
)
//***
// LogActions Tests
//***
func TestQuery_DescribeLogGroups(t *testing.T) {
origNewCWLogsClient := newCWLogsClient
t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
func TestHandleDescribeLogGroups_WhenLogGroupNamePrefixIsEmpty(t *testing.T) {
executor := &cloudWatchExecutor{}
var cli fakeCWLogsClient
logsClient := &FakeLogsClient{
Config: aws.Config{
Region: aws.String("default"),
newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return cli
}
t.Run("Empty log group name prefix", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
},
},
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "DescribeLogGroups",
"limit": 50,
}),
},
},
})
require.NoError(t, err)
require.NotNil(t, resp)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{
&data.Frame{
Name: "logGroups",
Fields: []*data.Field{
data.NewField("logGroupName", nil, []*string{
aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
}),
},
Meta: &data.FrameMeta{
PreferredVisualization: "logs",
},
},
}),
},
},
}, resp)
})
t.Run("Non-empty log group name prefix", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroups: cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
},
},
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "DescribeLogGroups",
"logGroupNamePrefix": "g",
}),
},
},
})
require.NoError(t, err)
require.NotNil(t, resp)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{
&data.Frame{
Name: "logGroups",
Fields: []*data.Field{
data.NewField("logGroupName", nil, []*string{
aws.String("group_a"), aws.String("group_b"), aws.String("group_c"),
}),
},
Meta: &data.FrameMeta{
PreferredVisualization: "logs",
},
},
}),
},
},
}, resp)
})
}
func TestQuery_GetLogGroupFields(t *testing.T) {
origNewCWLogsClient := newCWLogsClient
t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
var cli fakeCWLogsClient
newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return cli
}
cli = fakeCWLogsClient{
logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
},
}
params := simplejson.NewFromAny(map[string]interface{}{
"limit": 50,
const refID = "A"
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
RefId: refID,
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "GetLogGroupFields",
"logGroupName": "group_a",
"limit": 50,
}),
},
},
})
require.NoError(t, err)
require.NotNil(t, resp)
frame, err := executor.handleDescribeLogGroups(context.Background(), logsClient, params)
expectedField := data.NewField("logGroupName", nil, []*string{aws.String("group_a"), aws.String("group_b"), aws.String("group_c")})
expectedFrame := data.NewFrame("logGroups", expectedField)
assert.Equal(t, nil, err)
assert.Equal(t, expectedFrame, frame)
}
func TestHandleDescribeLogGroups_WhenLogGroupNamePrefixIsNotEmpty(t *testing.T) {
executor := &cloudWatchExecutor{}
logsClient := &FakeLogsClient{
Config: aws.Config{
Region: aws.String("default"),
expFrame := &data.Frame{
Name: refID,
Fields: []*data.Field{
data.NewField("name", nil, []*string{
aws.String("field_a"), aws.String("field_b"), aws.String("field_c"),
}),
data.NewField("percent", nil, []*int64{
aws.Int64(100), aws.Int64(30), aws.Int64(55),
}),
},
Meta: &data.FrameMeta{
PreferredVisualization: "logs",
},
}
params := simplejson.NewFromAny(map[string]interface{}{
"logGroupNamePrefix": "g",
})
frame, err := executor.handleDescribeLogGroups(context.Background(), logsClient, params)
expectedField := data.NewField("logGroupName", nil, []*string{aws.String("group_a"), aws.String("group_b"), aws.String("group_c")})
expectedFrame := data.NewFrame("logGroups", expectedField)
assert.Equal(t, nil, err)
assert.Equal(t, expectedFrame, frame)
expFrame.RefID = refID
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
refID: {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
RefId: refID,
},
},
}, resp)
}
func TestHandleGetLogGroupFields_WhenLogGroupNamePrefixIsNotEmpty(t *testing.T) {
executor := &cloudWatchExecutor{}
func TestQuery_StartQuery(t *testing.T) {
origNewCWLogsClient := newCWLogsClient
t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
logsClient := &FakeLogsClient{
Config: aws.Config{
Region: aws.String("default"),
},
var cli fakeCWLogsClient
newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return cli
}
params := simplejson.NewFromAny(map[string]interface{}{
"logGroupName": "group_a",
"limit": 50,
t.Run("invalid time range", func(t *testing.T) {
cli = fakeCWLogsClient{
logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
},
}
timeRange := &tsdb.TimeRange{
From: "1584873443000",
To: "1584700643000",
}
executor := newExecutor()
_, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "StartQuery",
"limit": 50,
"region": "default",
"queryString": "fields @message",
}),
},
},
})
require.Error(t, err)
assert.Equal(t, fmt.Errorf("invalid time range: start time must be before end time"), err)
})
frame, err := executor.handleGetLogGroupFields(context.Background(), logsClient, params, "A")
t.Run("valid time range", func(t *testing.T) {
const refID = "A"
cli = fakeCWLogsClient{
logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
},
}
expectedNameField := data.NewField("name", nil, []*string{aws.String("field_a"), aws.String("field_b"), aws.String("field_c")})
expectedPercentField := data.NewField("percent", nil, []*int64{aws.Int64(100), aws.Int64(30), aws.Int64(55)})
expectedFrame := data.NewFrame("A", expectedNameField, expectedPercentField)
expectedFrame.RefID = "A"
timeRange := &tsdb.TimeRange{
From: "1584700643000",
To: "1584873443000",
}
assert.Equal(t, nil, err)
assert.Equal(t, expectedFrame, frame)
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: []*tsdb.Query{
{
RefId: refID,
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "StartQuery",
"limit": 50,
"region": "default",
"queryString": "fields @message",
}),
},
},
})
require.NoError(t, err)
expFrame := data.NewFrame(
refID,
data.NewField("queryId", nil, []string{"abcd-efgh-ijkl-mnop"}),
)
expFrame.RefID = refID
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"Region": "default",
},
PreferredVisualization: "logs",
}
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
refID: {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
RefId: refID,
},
},
}, resp)
})
}
func TestExecuteStartQuery(t *testing.T) {
executor := &cloudWatchExecutor{}
func TestQuery_StopQuery(t *testing.T) {
origNewCWLogsClient := newCWLogsClient
t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
logsClient := &FakeLogsClient{
Config: aws.Config{
Region: aws.String("default"),
var cli fakeCWLogsClient
newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return cli
}
cli = fakeCWLogsClient{
logGroupFields: cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
},
}
@@ -102,109 +371,122 @@ func TestExecuteStartQuery(t *testing.T) {
To: "1584700643000",
}
params := simplejson.NewFromAny(map[string]interface{}{
"region": "default",
"limit": 50,
"queryString": "fields @message",
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
TimeRange: timeRange,
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "StopQuery",
"queryId": "abcd-efgh-ijkl-mnop",
}),
},
},
})
require.NoError(t, err)
response, err := executor.executeStartQuery(context.Background(), logsClient, params, timeRange)
var expectedResponse *cloudwatchlogs.StartQueryOutput = nil
assert.Equal(t, expectedResponse, response)
assert.Equal(t, fmt.Errorf("invalid time range: start time must be before end time"), err)
expFrame := &data.Frame{
Name: "StopQueryResponse",
Fields: []*data.Field{
data.NewField("success", nil, []bool{true}),
},
Meta: &data.FrameMeta{
PreferredVisualization: "logs",
},
}
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
},
},
}, resp)
}
func TestHandleStartQuery(t *testing.T) {
executor := &cloudWatchExecutor{}
logsClient := &FakeLogsClient{
Config: aws.Config{
Region: aws.String("default"),
},
}
timeRange := &tsdb.TimeRange{
From: "1584700643000",
To: "1584873443000",
}
params := simplejson.NewFromAny(map[string]interface{}{
"region": "default",
"limit": 50,
"queryString": "fields @message",
func TestQuery_GetQueryResults(t *testing.T) {
origNewCWLogsClient := newCWLogsClient
t.Cleanup(func() {
newCWLogsClient = origNewCWLogsClient
})
frame, err := executor.handleStartQuery(context.Background(), logsClient, params, timeRange, "A")
var cli fakeCWLogsClient
expectedField := data.NewField("queryId", nil, []string{"abcd-efgh-ijkl-mnop"})
expectedFrame := data.NewFrame("A", expectedField)
expectedFrame.RefID = "A"
expectedFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"Region": "default",
newCWLogsClient = func(sess *session.Session) cloudwatchlogsiface.CloudWatchLogsAPI {
return cli
}
const refID = "A"
cli = fakeCWLogsClient{
queryResults: cloudwatchlogs.GetQueryResultsOutput{
Results: [][]*cloudwatchlogs.ResultField{
{
{
Field: aws.String("@timestamp"),
Value: aws.String("2020-03-20 10:37:23.000"),
},
{
Field: aws.String("field_b"),
Value: aws.String("b_1"),
},
{
Field: aws.String("@ptr"),
Value: aws.String("abcdefg"),
},
},
{
{
Field: aws.String("@timestamp"),
Value: aws.String("2020-03-20 10:40:43.000"),
},
{
Field: aws.String("field_b"),
Value: aws.String("b_2"),
},
{
Field: aws.String("@ptr"),
Value: aws.String("hijklmnop"),
},
},
},
Statistics: &cloudwatchlogs.QueryStatistics{
BytesScanned: aws.Float64(512),
RecordsMatched: aws.Float64(256),
RecordsScanned: aws.Float64(1024),
},
Status: aws.String("Complete"),
},
}
assert.Equal(t, nil, err)
assert.Equal(t, expectedFrame, frame)
}
func TestHandleStopQuery(t *testing.T) {
executor := &cloudWatchExecutor{}
logsClient := &FakeLogsClient{
Config: aws.Config{
Region: aws.String("default"),
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
RefId: refID,
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "logAction",
"subtype": "GetQueryResults",
"queryId": "abcd-efgh-ijkl-mnop",
}),
},
},
}
params := simplejson.NewFromAny(map[string]interface{}{
"queryId": "abcd-efgh-ijkl-mnop",
})
frame, err := executor.handleStopQuery(context.Background(), logsClient, params)
expectedField := data.NewField("success", nil, []bool{true})
expectedFrame := data.NewFrame("StopQueryResponse", expectedField)
assert.Equal(t, nil, err)
assert.Equal(t, expectedFrame, frame)
}
func TestHandleGetQueryResults(t *testing.T) {
executor := &cloudWatchExecutor{}
logsClient := &FakeLogsClient{
Config: aws.Config{
Region: aws.String("default"),
},
}
params := simplejson.NewFromAny(map[string]interface{}{
"queryId": "abcd-efgh-ijkl-mnop",
})
frame, err := executor.handleGetQueryResults(context.Background(), logsClient, params, "A")
require.NoError(t, err)
timeA, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:37:23.000")
require.NoError(t, err)
timeB, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:40:43.000")
require.NoError(t, err)
expectedTimeField := data.NewField("@timestamp", nil, []*time.Time{
aws.Time(timeA), aws.Time(timeB),
})
expectedTimeField.SetConfig(&data.FieldConfig{DisplayName: "Time"})
expectedFieldB := data.NewField("field_b", nil, []*string{
time1, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:37:23.000")
require.NoError(t, err)
time2, err := time.Parse("2006-01-02 15:04:05.000", "2020-03-20 10:40:43.000")
require.NoError(t, err)
expField1 := data.NewField("@timestamp", nil, []*time.Time{
aws.Time(time1), aws.Time(time2),
})
expField1.SetConfig(&data.FieldConfig{DisplayName: "Time"})
expField2 := data.NewField("field_b", nil, []*string{
aws.String("b_1"), aws.String("b_2"),
})
expectedFrame := data.NewFrame("A", expectedTimeField, expectedFieldB)
expectedFrame.RefID = "A"
expectedFrame.Meta = &data.FrameMeta{
expFrame := data.NewFrame(refID, expField1, expField2)
expFrame.RefID = refID
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"Status": "Complete",
"Statistics": cloudwatchlogs.QueryStatistics{
@@ -213,9 +495,15 @@ func TestHandleGetQueryResults(t *testing.T) {
RecordsScanned: aws.Float64(1024),
},
},
PreferredVisualization: "logs",
}
assert.Equal(t, nil, err)
assert.ElementsMatch(t, expectedFrame.Fields, frame.Fields)
assert.Equal(t, expectedFrame.Meta, frame.Meta)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
refID: {
RefId: refID,
Dataframes: tsdb.NewDecodedDataFrames(data.Frames{expFrame}),
},
},
}, resp)
}

106
pkg/tsdb/cloudwatch/logs_client_mock.go
View File

@@ -1,106 +0,0 @@
package cloudwatch
import (
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
)
type FakeLogsClient struct {
cloudwatchlogsiface.CloudWatchLogsAPI
Config aws.Config
}
func (f FakeLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
return &cloudwatchlogs.DescribeLogGroupsOutput{
LogGroups: []*cloudwatchlogs.LogGroup{
{
LogGroupName: aws.String("group_a"),
},
{
LogGroupName: aws.String("group_b"),
},
{
LogGroupName: aws.String("group_c"),
},
},
}, nil
}
func (f FakeLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {
return &cloudwatchlogs.GetLogGroupFieldsOutput{
LogGroupFields: []*cloudwatchlogs.LogGroupField{
{
Name: aws.String("field_a"),
Percent: aws.Int64(100),
},
{
Name: aws.String("field_b"),
Percent: aws.Int64(30),
},
{
Name: aws.String("field_c"),
Percent: aws.Int64(55),
},
},
}, nil
}
func (f FakeLogsClient) StartQueryWithContext(ctx context.Context, input *cloudwatchlogs.StartQueryInput, option ...request.Option) (*cloudwatchlogs.StartQueryOutput, error) {
return &cloudwatchlogs.StartQueryOutput{
QueryId: aws.String("abcd-efgh-ijkl-mnop"),
}, nil
}
func (f FakeLogsClient) StopQueryWithContext(ctx context.Context, input *cloudwatchlogs.StopQueryInput, option ...request.Option) (*cloudwatchlogs.StopQueryOutput, error) {
return &cloudwatchlogs.StopQueryOutput{
Success: aws.Bool(true),
}, nil
}
func (f FakeLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) {
return &cloudwatchlogs.GetQueryResultsOutput{
Results: [][]*cloudwatchlogs.ResultField{
{
{
Field: aws.String("@timestamp"),
Value: aws.String("2020-03-20 10:37:23.000"),
},
{
Field: aws.String("field_b"),
Value: aws.String("b_1"),
},
{
Field: aws.String("@ptr"),
Value: aws.String("abcdefg"),
},
},
{
{
Field: aws.String("@timestamp"),
Value: aws.String("2020-03-20 10:40:43.000"),
},
{
Field: aws.String("field_b"),
Value: aws.String("b_2"),
},
{
Field: aws.String("@ptr"),
Value: aws.String("hijklmnop"),
},
},
},
Statistics: &cloudwatchlogs.QueryStatistics{
BytesScanned: aws.Float64(512),
RecordsMatched: aws.Float64(256),
RecordsScanned: aws.Float64(1024),
},
Status: aws.String("Complete"),
}, nil
}

97
pkg/tsdb/cloudwatch/metric_find_query.go
View File

@@ -12,7 +12,6 @@ import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/awsutil"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
@@ -310,8 +309,7 @@ func parseMultiSelectValue(input string) []string {
// Whenever this list is updated, the frontend list should also be updated.
// Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html
func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
const region = "default"
dsInfo := e.getDSInfo(region)
dsInfo := e.getDSInfo(defaultRegion)
profile := dsInfo.Profile
if cache, ok := regionCache.Load(profile); ok {
if cache2, ok2 := cache.([]suggestData); ok2 {
@@ -319,12 +317,12 @@ func (e *cloudWatchExecutor) handleGetRegions(ctx context.Context, parameters *s
}
}
err := e.ensureClientSession("default")
client, err := e.getEC2Client(defaultRegion)
if err != nil {
return nil, err
}
regions := knownRegions
r, err := e.ec2Svc.DescribeRegions(&ec2.DescribeRegionsInput{})
r, err := client.DescribeRegions(&ec2.DescribeRegionsInput{})
if err != nil {
// ignore error for backward compatibility
plog.Error("Failed to get regions", "error", err)
@@ -389,7 +387,7 @@ func (e *cloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *s
dsInfo := e.getDSInfo(region)
dsInfo.Namespace = namespace
if namespaceMetrics, err = e.getMetricsForCustomMetrics(region, e.getAllMetrics); err != nil {
if namespaceMetrics, err = e.getMetricsForCustomMetrics(region); err != nil {
return nil, errors.New("Unable to call AWS API")
}
}
@@ -418,7 +416,7 @@ func (e *cloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters
dsInfo := e.getDSInfo(region)
dsInfo.Namespace = namespace
if dimensionValues, err = e.getDimensionsForCustomMetrics(region, e.getAllMetrics); err != nil {
if dimensionValues, err = e.getDimensionsForCustomMetrics(region); err != nil {
return nil, errors.New("Unable to call AWS API")
}
}
@@ -483,31 +481,10 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param
return result, nil
}
func (e *cloudWatchExecutor) ensureClientSession(region string) error {
if e.ec2Svc == nil {
dsInfo := e.getDSInfo(region)
cfg, err := getAwsConfig(dsInfo)
if err != nil {
return fmt.Errorf("Failed to call ec2:getAwsConfig, %w", err)
}
sess, err := session.NewSession(cfg)
if err != nil {
return fmt.Errorf("Failed to call ec2:NewSession, %w", err)
}
e.ec2Svc = ec2.New(sess, cfg)
}
return nil
}
func (e *cloudWatchExecutor) handleGetEbsVolumeIds(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
region := parameters.Get("region").MustString()
instanceId := parameters.Get("instanceId").MustString()
err := e.ensureClientSession(region)
if err != nil {
return nil, err
}
instanceIds := aws.StringSlice(parseMultiSelectValue(instanceId))
instances, err := e.ec2DescribeInstances(region, nil, instanceIds)
if err != nil {
@@ -547,11 +524,6 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context,
}
}
err := e.ensureClientSession(region)
if err != nil {
return nil, err
}
instances, err := e.ec2DescribeInstances(region, filters, nil)
if err != nil {
return nil, err
@@ -610,32 +582,11 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(ctx context.Context,
return result, nil
}
func (e *cloudWatchExecutor) ensureRGTAClientSession(region string) error {
if e.rgtaSvc == nil {
dsInfo := e.getDSInfo(region)
cfg, err := getAwsConfig(dsInfo)
if err != nil {
return fmt.Errorf("Failed to call ec2:getAwsConfig, %w", err)
}
sess, err := session.NewSession(cfg)
if err != nil {
return fmt.Errorf("Failed to call ec2:NewSession, %w", err)
}
e.rgtaSvc = resourcegroupstaggingapi.New(sess, cfg)
}
return nil
}
func (e *cloudWatchExecutor) handleGetResourceArns(ctx context.Context, parameters *simplejson.Json, queryContext *tsdb.TsdbQuery) ([]suggestData, error) {
region := parameters.Get("region").MustString()
resourceType := parameters.Get("resourceType").MustString()
filterJson := parameters.Get("tags").MustMap()
err := e.ensureRGTAClientSession(region)
if err != nil {
return nil, err
}
var filters []*resourcegroupstaggingapi.TagFilter
for k, v := range filterJson {
if vv, ok := v.([]interface{}); ok {
@@ -706,8 +657,13 @@ func (e *cloudWatchExecutor) ec2DescribeInstances(region string, filters []*ec2.
InstanceIds: instanceIds,
}
client, err := e.getEC2Client(region)
if err != nil {
return nil, err
}
var resp ec2.DescribeInstancesOutput
if err := e.ec2Svc.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool {
if err := client.DescribeInstancesPages(params, func(page *ec2.DescribeInstancesOutput, lastPage bool) bool {
resp.Reservations = append(resp.Reservations, page.Reservations...)
return !lastPage
}); err != nil {
@@ -724,8 +680,13 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [
TagFilters: filters,
}
client, err := e.getRGTAClient(region)
if err != nil {
return nil, err
}
var resp resourcegroupstaggingapi.GetResourcesOutput
if err := e.rgtaSvc.GetResourcesPages(params,
if err := client.GetResourcesPages(params,
func(page *resourcegroupstaggingapi.GetResourcesOutput, lastPage bool) bool {
resp.ResourceTagMappingList = append(resp.ResourceTagMappingList, page.ResourceTagMappingList...)
return !lastPage
@@ -737,27 +698,18 @@ func (e *cloudWatchExecutor) resourceGroupsGetResources(region string, filters [
}
func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetricsOutput, error) {
dsInfo := e.getDSInfo(region)
creds, err := getCredentials(dsInfo)
client, err := e.getCWClient(region)
if err != nil {
return cloudwatch.ListMetricsOutput{}, err
}
cfg := &aws.Config{
Region: aws.String(dsInfo.Region),
Credentials: creds,
}
sess, err := session.NewSession(cfg)
if err != nil {
return cloudwatch.ListMetricsOutput{}, err
}
svc := cloudwatch.New(sess, cfg)
dsInfo := e.getDSInfo(region)
params := &cloudwatch.ListMetricsInput{
Namespace: aws.String(dsInfo.Namespace),
}
var resp cloudwatch.ListMetricsOutput
err = svc.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
err = client.ListMetricsPages(params, func(page *cloudwatch.ListMetricsOutput, lastPage bool) bool {
metrics.MAwsCloudWatchListMetrics.Inc()
metrics, err := awsutil.ValuesAtPath(page, "Metrics")
if err != nil {
@@ -769,12 +721,13 @@ func (e *cloudWatchExecutor) getAllMetrics(region string) (cloudwatch.ListMetric
}
return !lastPage
})
return resp, err
}
var metricsCacheLock sync.Mutex
func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMetrics func(string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) {
func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string) ([]string, error) {
metricsCacheLock.Lock()
defer metricsCacheLock.Unlock()
@@ -794,7 +747,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMet
if customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) {
return customMetricsMetricsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil
}
result, err := getAllMetrics(region)
result, err := e.getAllMetrics(region)
if err != nil {
return []string{}, err
}
@@ -813,7 +766,7 @@ func (e *cloudWatchExecutor) getMetricsForCustomMetrics(region string, getAllMet
var dimensionsCacheLock sync.Mutex
func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string, getAllMetrics func(string) (cloudwatch.ListMetricsOutput, error)) ([]string, error) {
func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string) ([]string, error) {
dimensionsCacheLock.Lock()
defer dimensionsCacheLock.Unlock()
@@ -833,7 +786,7 @@ func (e *cloudWatchExecutor) getDimensionsForCustomMetrics(region string, getAll
if customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Expire.After(time.Now()) {
return customMetricsDimensionsMap[dsInfo.Profile][dsInfo.Region][dsInfo.Namespace].Cache, nil
}
result, err := getAllMetrics(region)
result, err := e.getAllMetrics(region)
if err != nil {
return []string{}, err
}

689
pkg/tsdb/cloudwatch/metric_find_query_test.go
View File

@@ -5,241 +5,236 @@ import (
"testing"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/client"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
type mockedEc2 struct {
ec2iface.EC2API
Resp ec2.DescribeInstancesOutput
RespRegions ec2.DescribeRegionsOutput
}
type mockedRGTA struct {
resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
Resp resourcegroupstaggingapi.GetResourcesOutput
}
func (m mockedEc2) DescribeInstancesPages(in *ec2.DescribeInstancesInput, fn func(*ec2.DescribeInstancesOutput, bool) bool) error {
fn(&m.Resp, true)
return nil
}
func (m mockedEc2) DescribeRegions(in *ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
return &m.RespRegions, nil
}
func (m mockedRGTA) GetResourcesPages(in *resourcegroupstaggingapi.GetResourcesInput, fn func(*resourcegroupstaggingapi.GetResourcesOutput, bool) bool) error {
fn(&m.Resp, true)
return nil
}
func TestCloudWatchMetrics(t *testing.T) {
t.Run("When calling getMetricsForCustomMetrics", func(t *testing.T) {
const region = "us-east-1"
e := &cloudWatchExecutor{
DataSource: &models.DataSource{
Database: "default",
JsonData: simplejson.NewFromAny(map[string]interface{}{
"Region": region,
}),
},
}
f := func(region string) (cloudwatch.ListMetricsOutput, error) {
return cloudwatch.ListMetricsOutput{
Metrics: []*cloudwatch.Metric{
{
MetricName: aws.String("Test_MetricName"),
Dimensions: []*cloudwatch.Dimension{
{
Name: aws.String("Test_DimensionName"),
},
},
},
},
}, nil
}
metrics, err := e.getMetricsForCustomMetrics(region, f)
require.NoError(t, err)
assert.Contains(t, metrics, "Test_MetricName")
func TestQuery_Metrics(t *testing.T) {
origNewCWClient := newCWClient
t.Cleanup(func() {
newCWClient = origNewCWClient
})
t.Run("When calling getDimensionsForCustomMetrics", func(t *testing.T) {
const region = "us-east-1"
e := &cloudWatchExecutor{
DataSource: &models.DataSource{
Database: "default",
JsonData: simplejson.NewFromAny(map[string]interface{}{
"Region": region,
}),
},
}
f := func(region string) (cloudwatch.ListMetricsOutput, error) {
return cloudwatch.ListMetricsOutput{
Metrics: []*cloudwatch.Metric{
{
MetricName: aws.String("Test_MetricName"),
Dimensions: []*cloudwatch.Dimension{
{
Name: aws.String("Test_DimensionName"),
},
},
},
},
}, nil
}
dimensionKeys, err := e.getDimensionsForCustomMetrics(region, f)
require.NoError(t, err)
var client fakeCWClient
assert.Contains(t, dimensionKeys, "Test_DimensionName")
})
newCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
return client
}
t.Run("When calling handleGetRegions", func(t *testing.T) {
executor := &cloudWatchExecutor{
ec2Svc: mockedEc2{RespRegions: ec2.DescribeRegionsOutput{
Regions: []*ec2.Region{
{
RegionName: aws.String("ap-northeast-2"),
},
},
}},
}
jsonData := simplejson.New()
jsonData.Set("defaultRegion", "default")
executor.DataSource = &models.DataSource{
JsonData: jsonData,
SecureJsonData: securejsondata.SecureJsonData{},
}
result, err := executor.handleGetRegions(context.Background(), simplejson.New(), &tsdb.TsdbQuery{})
require.NoError(t, err)
assert.Equal(t, "af-south-1", result[0].Text)
assert.Equal(t, "ap-east-1", result[1].Text)
assert.Equal(t, "ap-northeast-1", result[2].Text)
assert.Equal(t, "ap-northeast-2", result[3].Text)
})
t.Run("When calling handleGetEc2InstanceAttribute", func(t *testing.T) {
executor := &cloudWatchExecutor{
ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{
Reservations: []*ec2.Reservation{
{
Instances: []*ec2.Instance{
{
InstanceId: aws.String("i-12345678"),
Tags: []*ec2.Tag{
{
Key: aws.String("Environment"),
Value: aws.String("production"),
},
},
},
},
},
},
}},
}
json := simplejson.New()
json.Set("region", "us-east-1")
json.Set("attributeName", "InstanceId")
filters := make(map[string]interface{})
filters["tag:Environment"] = []string{"production"}
json.Set("filters", filters)
result, err := executor.handleGetEc2InstanceAttribute(context.Background(), json, &tsdb.TsdbQuery{})
require.NoError(t, err)
assert.Equal(t, "i-12345678", result[0].Text)
})
t.Run("When calling handleGetEbsVolumeIds", func(t *testing.T) {
executor := &cloudWatchExecutor{
ec2Svc: mockedEc2{Resp: ec2.DescribeInstancesOutput{
Reservations: []*ec2.Reservation{
{
Instances: []*ec2.Instance{
{
InstanceId: aws.String("i-1"),
BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-1")}},
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-2")}},
},
},
{
InstanceId: aws.String("i-2"),
BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-1")}},
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-2")}},
},
},
},
},
{
Instances: []*ec2.Instance{
{
InstanceId: aws.String("i-3"),
BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-1")}},
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-2")}},
},
},
{
InstanceId: aws.String("i-4"),
BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-1")}},
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-2")}},
},
},
},
},
},
}},
}
json := simplejson.New()
json.Set("region", "us-east-1")
json.Set("instanceId", "{i-1, i-2, i-3, i-4}")
result, err := executor.handleGetEbsVolumeIds(context.Background(), json, &tsdb.TsdbQuery{})
require.NoError(t, err)
require.Len(t, result, 8)
assert.Equal(t, "vol-1-1", result[0].Text)
assert.Equal(t, "vol-1-2", result[1].Text)
assert.Equal(t, "vol-2-1", result[2].Text)
assert.Equal(t, "vol-2-2", result[3].Text)
assert.Equal(t, "vol-3-1", result[4].Text)
assert.Equal(t, "vol-3-2", result[5].Text)
assert.Equal(t, "vol-4-1", result[6].Text)
assert.Equal(t, "vol-4-2", result[7].Text)
})
t.Run("When calling handleGetResourceArns", func(t *testing.T) {
executor := &cloudWatchExecutor{
rgtaSvc: mockedRGTA{
Resp: resourcegroupstaggingapi.GetResourcesOutput{
ResourceTagMappingList: []*resourcegroupstaggingapi.ResourceTagMapping{
t.Run("Custom metrics", func(t *testing.T) {
client = fakeCWClient{
metrics: []*cloudwatch.Metric{
{
MetricName: aws.String("Test_MetricName"),
Dimensions: []*cloudwatch.Dimension{
{
ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567"),
Tags: []*resourcegroupstaggingapi.Tag{
Name: aws.String("Test_DimensionName"),
},
},
},
},
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "metrics",
"region": "us-east-1",
"namespace": "custom",
}),
},
},
})
require.NoError(t, err)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": 1,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Key: aws.String("Environment"),
Value: aws.String("production"),
Text: "text",
},
{
Text: "value",
},
},
Rows: []tsdb.RowValues{
{
"Test_MetricName",
"Test_MetricName",
},
},
},
},
},
},
}, resp)
})
t.Run("Dimension keys for custom metrics", func(t *testing.T) {
client = fakeCWClient{
metrics: []*cloudwatch.Metric{
{
MetricName: aws.String("Test_MetricName"),
Dimensions: []*cloudwatch.Dimension{
{
ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321"),
Tags: []*resourcegroupstaggingapi.Tag{
Name: aws.String("Test_DimensionName"),
},
},
},
},
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "dimension_keys",
"region": "us-east-1",
"namespace": "custom",
}),
},
},
})
require.NoError(t, err)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": 1,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: []tsdb.RowValues{
{
"Test_DimensionName",
"Test_DimensionName",
},
},
},
},
},
},
}, resp)
})
}
func TestQuery_Regions(t *testing.T) {
origNewEC2Client := newEC2Client
t.Cleanup(func() {
newEC2Client = origNewEC2Client
})
var cli fakeEC2Client
newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
return cli
}
t.Run("An extra region", func(t *testing.T) {
const regionName = "xtra-region"
cli = fakeEC2Client{
regions: []string{regionName},
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "regions",
"region": "us-east-1",
"namespace": "custom",
}),
},
},
})
require.NoError(t, err)
rows := []tsdb.RowValues{}
for _, region := range knownRegions {
rows = append(rows, []interface{}{
region,
region,
})
}
rows = append(rows, []interface{}{
regionName,
regionName,
})
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": len(knownRegions) + 1,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: rows,
},
},
},
},
}, resp)
})
}
func TestQuery_InstanceAttributes(t *testing.T) {
origNewEC2Client := newEC2Client
t.Cleanup(func() {
newEC2Client = origNewEC2Client
})
var cli fakeEC2Client
newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
return cli
}
t.Run("Get instance ID", func(t *testing.T) {
const instanceID = "i-12345678"
cli = fakeEC2Client{
reservations: []*ec2.Reservation{
{
Instances: []*ec2.Instance{
{
InstanceId: aws.String(instanceID),
Tags: []*ec2.Tag{
{
Key: aws.String("Environment"),
Value: aws.String("production"),
@@ -250,33 +245,257 @@ func TestCloudWatchMetrics(t *testing.T) {
},
},
}
json := simplejson.New()
json.Set("region", "us-east-1")
json.Set("resourceType", "ec2:instance")
tags := make(map[string]interface{})
tags["Environment"] = []string{"production"}
json.Set("tags", tags)
result, err := executor.handleGetResourceArns(context.Background(), json, &tsdb.TsdbQuery{})
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "ec2_instance_attribute",
"region": "us-east-1",
"attributeName": "InstanceId",
"filters": map[string]interface{}{
"tag:Environment": []string{"production"},
},
}),
},
},
})
require.NoError(t, err)
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", result[0].Text)
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567", result[0].Value)
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", result[1].Text)
assert.Equal(t, "arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321", result[1].Value)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": 1,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: []tsdb.RowValues{
{
instanceID,
instanceID,
},
},
},
},
},
},
}, resp)
})
}
func TestParseMultiSelectValue(t *testing.T) {
values := parseMultiSelectValue(" i-someInstance ")
assert.Equal(t, []string{"i-someInstance"}, values)
func TestQuery_EBSVolumeIDs(t *testing.T) {
origNewEC2Client := newEC2Client
t.Cleanup(func() {
newEC2Client = origNewEC2Client
})
values = parseMultiSelectValue("{i-05}")
assert.Equal(t, []string{"i-05"}, values)
var cli fakeEC2Client
values = parseMultiSelectValue(" {i-01, i-03, i-04} ")
assert.Equal(t, []string{"i-01", "i-03", "i-04"}, values)
newEC2Client = func(client.ConfigProvider) ec2iface.EC2API {
return cli
}
values = parseMultiSelectValue("i-{01}")
assert.Equal(t, []string{"i-{01}"}, values)
t.Run("", func(t *testing.T) {
const instanceIDs = "{i-1, i-2, i-3}"
cli = fakeEC2Client{
reservations: []*ec2.Reservation{
{
Instances: []*ec2.Instance{
{
InstanceId: aws.String("i-1"),
BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-1")}},
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-1-2")}},
},
},
{
InstanceId: aws.String("i-2"),
BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-1")}},
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-2-2")}},
},
},
},
},
{
Instances: []*ec2.Instance{
{
InstanceId: aws.String("i-3"),
BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-1")}},
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-3-2")}},
},
},
{
InstanceId: aws.String("i-4"),
BlockDeviceMappings: []*ec2.InstanceBlockDeviceMapping{
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-1")}},
{Ebs: &ec2.EbsInstanceBlockDevice{VolumeId: aws.String("vol-4-2")}},
},
},
},
},
},
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "ebs_volume_ids",
"region": "us-east-1",
"instanceId": instanceIDs,
}),
},
},
})
require.NoError(t, err)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": 6,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: []tsdb.RowValues{
{
"vol-1-1",
"vol-1-1",
},
{
"vol-1-2",
"vol-1-2",
},
{
"vol-2-1",
"vol-2-1",
},
{
"vol-2-2",
"vol-2-2",
},
{
"vol-3-1",
"vol-3-1",
},
{
"vol-3-2",
"vol-3-2",
},
},
},
},
},
},
}, resp)
})
}
func TestQuery_ResourceARNs(t *testing.T) {
origNewRGTAClient := newRGTAClient
t.Cleanup(func() {
newRGTAClient = origNewRGTAClient
})
var cli fakeRGTAClient
newRGTAClient = func(client.ConfigProvider) resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI {
return cli
}
t.Run("", func(t *testing.T) {
cli = fakeRGTAClient{
tagMapping: []*resourcegroupstaggingapi.ResourceTagMapping{
{
ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567"),
Tags: []*resourcegroupstaggingapi.Tag{
{
Key: aws.String("Environment"),
Value: aws.String("production"),
},
},
},
{
ResourceARN: aws.String("arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321"),
Tags: []*resourcegroupstaggingapi.Tag{
{
Key: aws.String("Environment"),
Value: aws.String("production"),
},
},
},
},
}
executor := newExecutor()
resp, err := executor.Query(context.Background(), fakeDataSource(), &tsdb.TsdbQuery{
Queries: []*tsdb.Query{
{
Model: simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "resource_arns",
"region": "us-east-1",
"resourceType": "ec2:instance",
"tags": map[string]interface{}{
"Environment": []string{"production"},
},
}),
},
},
})
require.NoError(t, err)
assert.Equal(t, &tsdb.Response{
Results: map[string]*tsdb.QueryResult{
"": {
Meta: simplejson.NewFromAny(map[string]interface{}{
"rowCount": 2,
}),
Tables: []*tsdb.Table{
{
Columns: []tsdb.TableColumn{
{
Text: "text",
},
{
Text: "value",
},
},
Rows: []tsdb.RowValues{
{
"arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567",
"arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567",
},
{
"arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321",
"arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321",
},
},
},
},
},
},
}, resp)
})
}

2
pkg/tsdb/cloudwatch/query_transformer_test.go
View File

@@ -10,7 +10,7 @@ import (
func TestQueryTransformer(t *testing.T) {
Convey("TestQueryTransformer", t, func() {
Convey("when transforming queries", func() {
executor := &cloudWatchExecutor{}
executor := newExecutor()
Convey("one cloudwatchQuery is generated when its request query has one stat", func() {
requestQueries := []*requestQuery{
{

133
pkg/tsdb/cloudwatch/test_utils.go Normal file
View File

@@ -0,0 +1,133 @@
package cloudwatch
import (
"context"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/request"
"github.com/aws/aws-sdk-go/service/cloudwatch"
"github.com/aws/aws-sdk-go/service/cloudwatch/cloudwatchiface"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/ec2/ec2iface"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi/resourcegroupstaggingapiiface"
"github.com/grafana/grafana/pkg/components/securejsondata"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/models"
)
func fakeDataSource() *models.DataSource {
jsonData := simplejson.New()
jsonData.Set("defaultRegion", "default")
return &models.DataSource{
Id: 1,
Database: "default",
JsonData: jsonData,
SecureJsonData: securejsondata.SecureJsonData{},
}
}
type fakeCWLogsClient struct {
cloudwatchlogsiface.CloudWatchLogsAPI
logGroups cloudwatchlogs.DescribeLogGroupsOutput
logGroupFields cloudwatchlogs.GetLogGroupFieldsOutput
queryResults cloudwatchlogs.GetQueryResultsOutput
}
func (m fakeCWLogsClient) GetQueryResultsWithContext(ctx context.Context, input *cloudwatchlogs.GetQueryResultsInput, option ...request.Option) (*cloudwatchlogs.GetQueryResultsOutput, error) {
return &m.queryResults, nil
}
func (m fakeCWLogsClient) StartQueryWithContext(ctx context.Context, input *cloudwatchlogs.StartQueryInput, option ...request.Option) (*cloudwatchlogs.StartQueryOutput, error) {
return &cloudwatchlogs.StartQueryOutput{
QueryId: aws.String("abcd-efgh-ijkl-mnop"),
}, nil
}
func (m fakeCWLogsClient) StopQueryWithContext(ctx context.Context, input *cloudwatchlogs.StopQueryInput, option ...request.Option) (*cloudwatchlogs.StopQueryOutput, error) {
return &cloudwatchlogs.StopQueryOutput{
Success: aws.Bool(true),
}, nil
}
func (m fakeCWLogsClient) DescribeLogGroupsWithContext(ctx context.Context, input *cloudwatchlogs.DescribeLogGroupsInput, option ...request.Option) (*cloudwatchlogs.DescribeLogGroupsOutput, error) {
return &m.logGroups, nil
}
func (m fakeCWLogsClient) GetLogGroupFieldsWithContext(ctx context.Context, input *cloudwatchlogs.GetLogGroupFieldsInput, option ...request.Option) (*cloudwatchlogs.GetLogGroupFieldsOutput, error) {
return &m.logGroupFields, nil
}
type fakeCWClient struct {
cloudwatchiface.CloudWatchAPI
metrics []*cloudwatch.Metric
}
func (c fakeCWClient) ListMetricsPages(input *cloudwatch.ListMetricsInput, fn func(*cloudwatch.ListMetricsOutput, bool) bool) error {
fn(&cloudwatch.ListMetricsOutput{
Metrics: c.metrics,
}, true)
return nil
}
type fakeEC2Client struct {
ec2iface.EC2API
regions []string
reservations []*ec2.Reservation
}
func (c fakeEC2Client) DescribeRegions(*ec2.DescribeRegionsInput) (*ec2.DescribeRegionsOutput, error) {
regions := []*ec2.Region{}
for _, region := range c.regions {
regions = append(regions, &ec2.Region{
RegionName: aws.String(region),
})
}
return &ec2.DescribeRegionsOutput{
Regions: regions,
}, nil
}
func (c fakeEC2Client) DescribeInstancesPages(in *ec2.DescribeInstancesInput,
fn func(*ec2.DescribeInstancesOutput, bool) bool) error {
reservations := []*ec2.Reservation{}
for _, r := range c.reservations {
instances := []*ec2.Instance{}
for _, inst := range r.Instances {
if len(in.InstanceIds) == 0 {
instances = append(instances, inst)
continue
}
for _, id := range in.InstanceIds {
if *inst.InstanceId == *id {
instances = append(instances, inst)
}
}
}
reservation := &ec2.Reservation{Instances: instances}
reservations = append(reservations, reservation)
}
fn(&ec2.DescribeInstancesOutput{
Reservations: reservations,
}, true)
return nil
}
type fakeRGTAClient struct {
resourcegroupstaggingapiiface.ResourceGroupsTaggingAPIAPI
tagMapping []*resourcegroupstaggingapi.ResourceTagMapping
}
func (c fakeRGTAClient) GetResourcesPages(in *resourcegroupstaggingapi.GetResourcesInput,
fn func(*resourcegroupstaggingapi.GetResourcesOutput, bool) bool) error {
fn(&resourcegroupstaggingapi.GetResourcesOutput{
ResourceTagMappingList: c.tagMapping,
}, true)
return nil
}

2
pkg/tsdb/cloudwatch/time_series_query_test.go
View File

@@ -9,7 +9,7 @@ import (
)
func TestTimeSeriesQuery(t *testing.T) {
executor := &cloudWatchExecutor{}
executor := newExecutor()
t.Run("End time before start time should result in error", func(t *testing.T) {
_, err := executor.executeTimeSeriesQuery(context.TODO(), &tsdb.TsdbQuery{TimeRange: tsdb.NewTimeRange("now-1h", "now-2h")})