IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CloudWatch: replace metricFindQueries with CallResourceHandler (#41571)

Browse Source
This commit is contained in:
Isabella Siu 2022-02-16 14:28:26 -05:00 committed by GitHub
parent b01a56c2b7
commit 50a53ef58b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 301 additions and 544 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

73
pkg/tests/api/metrics/api_metrics_test.go
View File

@ -32,7 +32,7 @@ func TestQueryCloudWatchMetrics(t *testing.T) {
grafDir, cfgPath := testinfra.CreateGrafDir(t)
addr, sqlStore := testinfra.StartGrafana(t, grafDir, cfgPath)
setUpDatabase(t, sqlStore)
setUpDatabase(t, sqlStore, "metrics")
origNewCWClient := cloudwatch.NewCWClient
t.Cleanup(func() {
@ -56,47 +56,27 @@ func TestQueryCloudWatchMetrics(t *testing.T) {
},
},
}
result := getCWMetrics(t, 1, addr)
req := dtos.MetricRequest{
Queries: []*simplejson.Json{
simplejson.NewFromAny(map[string]interface{}{
"type": "metricFindQuery",
"subtype": "metrics",
"region": "us-east-1",
"namespace": "custom",
"datasourceId": 1,
}),
},
type suggestData struct {
Text string
Value string
Label string
}
result := makeCWRequest(t, req, addr)
dataFrames := data.Frames{
&data.Frame{
RefID: "A",
Fields: []*data.Field{
data.NewField("text", nil, []string{"Test_MetricName"}),
data.NewField("value", nil, []string{"Test_MetricName"}),
},
Meta: &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": float64(1),
},
},
},
expect := []suggestData{
{Text: "Test_MetricName", Value: "Test_MetricName", Label: "Test_MetricName"},
}
expect := backend.NewQueryDataResponse()
expect.Responses["A"] = backend.DataResponse{
Frames: dataFrames,
}
assert.Equal(t, *expect, result)
actual := []suggestData{}
err := json.Unmarshal(result, &actual)
require.NoError(t, err)
assert.Equal(t, expect, actual)
})
}
func TestQueryCloudWatchLogs(t *testing.T) {
grafDir, cfgPath := testinfra.CreateGrafDir(t)
addr, store := testinfra.StartGrafana(t, grafDir, cfgPath)
setUpDatabase(t, store)
setUpDatabase(t, store, "logs")
origNewCWLogsClient := cloudwatch.NewCWLogsClient
t.Cleanup(func() {
@ -141,6 +121,28 @@ func TestQueryCloudWatchLogs(t *testing.T) {
})
}
func getCWMetrics(t *testing.T, datasourceId int, addr string) []byte {
t.Helper()
u := fmt.Sprintf("http://%s/api/datasources/%v/resources/metrics?region=us-east-1&namespace=custom", addr, datasourceId)
t.Logf("Making GET request to %s", u)
// nolint:gosec
resp, err := http.Get(u)
require.NoError(t, err)
require.NotNil(t, resp)
t.Cleanup(func() {
err := resp.Body.Close()
assert.NoError(t, err)
})
buf := bytes.Buffer{}
_, err = io.Copy(&buf, resp.Body)
require.NoError(t, err)
require.Equal(t, 200, resp.StatusCode)
return buf.Bytes()
}
func makeCWRequest(t *testing.T, req dtos.MetricRequest, addr string) backend.QueryDataResponse {
t.Helper()
@ -171,12 +173,13 @@ func makeCWRequest(t *testing.T, req dtos.MetricRequest, addr string) backend.Qu
return tr
}
func setUpDatabase(t *testing.T, store *sqlstore.SQLStore) {
func setUpDatabase(t *testing.T, store *sqlstore.SQLStore, uid string) {
t.Helper()
err := store.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error {
_, err := sess.Insert(&models.DataSource{
Id: 1,
Id: 1,
Uid: uid,
// This will be the ID of the main org
OrgId: 2,
Name: "Test",

13
pkg/tsdb/cloudwatch/cloudwatch.go
View File

@ -23,6 +23,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/datasource"
"github.com/grafana/grafana-plugin-sdk-go/backend/instancemgmt"
"github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/httpclient"
@ -83,11 +84,13 @@ type SessionCache interface {
}
func newExecutor(im instancemgmt.InstanceManager, cfg *setting.Cfg, sessions SessionCache) *cloudWatchExecutor {
return &cloudWatchExecutor{
cwe := &cloudWatchExecutor{
im: im,
cfg: cfg,
sessions: sessions,
}
cwe.resourceHandler = httpadapter.New(cwe.newResourceMux())
return cwe
}
func NewInstanceSettings(httpClientProvider httpclient.Provider) datasource.InstanceFactoryFunc {
@ -158,6 +161,12 @@ type cloudWatchExecutor struct {
im instancemgmt.InstanceManager
cfg *setting.Cfg
sessions SessionCache
resourceHandler backend.CallResourceHandler
}
func (e *cloudWatchExecutor) CallResource(ctx context.Context, req *backend.CallResourceRequest, sender backend.CallResourceResponseSender) error {
return e.resourceHandler.CallResource(ctx, req, sender)
}
func (e *cloudWatchExecutor) newSession(pluginCtx backend.PluginContext, region string) (*session.Session, error) {
@ -283,8 +292,6 @@ func (e *cloudWatchExecutor) QueryData(ctx context.Context, req *backend.QueryDa
var result *backend.QueryDataResponse
switch queryType {
case "metricFindQuery":
result, err = e.executeMetricFindQuery(req.PluginContext, model, q)
case "annotationQuery":
result, err = e.executeAnnotationQuery(req.PluginContext, model, q)
case "logAction":

174
pkg/tsdb/cloudwatch/metric_find_query.go
View File

@ -1,8 +1,10 @@
package cloudwatch
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"reflect"
"sort"
"strings"
@ -15,15 +17,14 @@ import (
"github.com/aws/aws-sdk-go/service/ec2"
"github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/metrics"
"github.com/grafana/grafana/pkg/util/errutil"
)
type suggestData struct {
Text string
Value string
Text string `json:"text"`
Value string `json:"value"`
Label string `json:"label,omitempty"`
}
type customMetricsCache struct {
@ -36,61 +37,6 @@ var customMetricsDimensionsMap = make(map[string]map[string]map[string]*customMe
var regionCache sync.Map
func (e *cloudWatchExecutor) executeMetricFindQuery(pluginCtx backend.PluginContext, model *simplejson.Json, query backend.DataQuery) (*backend.QueryDataResponse, error) {
subType := model.Get("subtype").MustString()
var data []suggestData
var err error
switch subType {
case "regions":
data, err = e.handleGetRegions(pluginCtx)
case "namespaces":
data, err = e.handleGetNamespaces(pluginCtx)
case "metrics":
data, err = e.handleGetMetrics(pluginCtx, model)
case "all_metrics":
data, err = e.handleGetAllMetrics()
case "dimension_keys":
data, err = e.handleGetDimensions(pluginCtx, model)
case "dimension_values":
data, err = e.handleGetDimensionValues(pluginCtx, model)
case "ebs_volume_ids":
data, err = e.handleGetEbsVolumeIds(pluginCtx, model)
case "ec2_instance_attribute":
data, err = e.handleGetEc2InstanceAttribute(pluginCtx, model)
case "resource_arns":
data, err = e.handleGetResourceArns(pluginCtx, model)
}
if err != nil {
return nil, err
}
resp := backend.NewQueryDataResponse()
respD := resp.Responses[query.RefID]
respD.Frames = append(respD.Frames, transformToTable(data))
resp.Responses[query.RefID] = respD
return resp, nil
}
func transformToTable(d []suggestData) *data.Frame {
frame := data.NewFrame("",
data.NewField("text", nil, []string{}),
data.NewField("value", nil, []string{}))
for _, r := range d {
frame.AppendRow(r.Text, r.Value)
}
frame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": len(d),
},
}
return frame
}
func parseMultiSelectValue(input string) []string {
trimmedInput := strings.TrimSpace(input)
if strings.HasPrefix(trimmedInput, "{") {
@ -107,7 +53,7 @@ func parseMultiSelectValue(input string) []string {
// Whenever this list is updated, the frontend list should also be updated.
// Please update the region list in public/app/plugins/datasource/cloudwatch/partials/config.html
func (e *cloudWatchExecutor) handleGetRegions(pluginCtx backend.PluginContext) ([]suggestData, error) {
func (e *cloudWatchExecutor) handleGetRegions(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
dsInfo, err := e.getDSInfo(pluginCtx)
if err != nil {
return nil, err
@ -149,14 +95,14 @@ func (e *cloudWatchExecutor) handleGetRegions(pluginCtx backend.PluginContext) (
result := make([]suggestData, 0)
for _, region := range regions {
result = append(result, suggestData{Text: region, Value: region})
result = append(result, suggestData{Text: region, Value: region, Label: region})
}
regionCache.Store(profile, result)
return result, nil
}
func (e *cloudWatchExecutor) handleGetNamespaces(pluginCtx backend.PluginContext) ([]suggestData, error) {
func (e *cloudWatchExecutor) handleGetNamespaces(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
var keys []string
for key := range metricsMap {
keys = append(keys, key)
@ -175,15 +121,15 @@ func (e *cloudWatchExecutor) handleGetNamespaces(pluginCtx backend.PluginContext
result := make([]suggestData, 0)
for _, key := range keys {
result = append(result, suggestData{Text: key, Value: key})
result = append(result, suggestData{Text: key, Value: key, Label: key})
}
return result, nil
}
func (e *cloudWatchExecutor) handleGetMetrics(pluginCtx backend.PluginContext, parameters *simplejson.Json) ([]suggestData, error) {
region := parameters.Get("region").MustString()
namespace := parameters.Get("namespace").MustString()
func (e *cloudWatchExecutor) handleGetMetrics(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
region := parameters.Get("region")
namespace := parameters.Get("namespace")
var namespaceMetrics []string
if !isCustomMetrics(namespace) {
@ -201,32 +147,40 @@ func (e *cloudWatchExecutor) handleGetMetrics(pluginCtx backend.PluginContext, p
result := make([]suggestData, 0)
for _, name := range namespaceMetrics {
result = append(result, suggestData{Text: name, Value: name})
result = append(result, suggestData{Text: name, Value: name, Label: name})
}
return result, nil
}
// handleGetAllMetrics returns a slice of suggestData structs with metric and its namespace
func (e *cloudWatchExecutor) handleGetAllMetrics() ([]suggestData, error) {
func (e *cloudWatchExecutor) handleGetAllMetrics(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
result := make([]suggestData, 0)
for namespace, metrics := range metricsMap {
for _, metric := range metrics {
result = append(result, suggestData{Text: namespace, Value: metric})
result = append(result, suggestData{Text: namespace, Value: metric, Label: namespace})
}
}
return result, nil
}
// handleGetDimensions returns a slice of suggestData structs with dimension keys.
// handleGetDimensionKeys returns a slice of suggestData structs with dimension keys.
// If a dimension filters parameter is specified, a new api call to list metrics will be issued to load dimension keys for the given filter.
// If no dimension filter is specified, dimension keys will be retrieved from the hard coded map in this file.
func (e *cloudWatchExecutor) handleGetDimensions(pluginCtx backend.PluginContext, parameters *simplejson.Json) ([]suggestData, error) {
region := parameters.Get("region").MustString()
namespace := parameters.Get("namespace").MustString()
metricName := parameters.Get("metricName").MustString("")
dimensionFilters := parameters.Get("dimensionFilters").MustMap()
func (e *cloudWatchExecutor) handleGetDimensionKeys(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
region := parameters.Get("region")
namespace := parameters.Get("namespace")
metricName := parameters.Get("metricName")
dimensionFilterJson := parameters.Get("dimensionFilters")
dimensionFilters := map[string]interface{}{}
if dimensionFilterJson != "" {
err := json.Unmarshal([]byte(dimensionFilterJson), &dimensionFilters)
if err != nil {
return nil, fmt.Errorf("error unmarshaling dimensionFilters: %v", err)
}
}
var dimensionValues []string
if !isCustomMetrics(namespace) {
@ -303,7 +257,7 @@ func (e *cloudWatchExecutor) handleGetDimensions(pluginCtx backend.PluginContext
result := make([]suggestData, 0)
for _, name := range dimensionValues {
result = append(result, suggestData{Text: name, Value: name})
result = append(result, suggestData{Text: name, Value: name, Label: name})
}
return result, nil
@ -311,12 +265,18 @@ func (e *cloudWatchExecutor) handleGetDimensions(pluginCtx backend.PluginContext
// handleGetDimensionValues returns a slice of suggestData structs with dimension values.
// A call to the list metrics api is issued to retrieve the dimension values. All parameters are used as input args to the list metrics call.
func (e *cloudWatchExecutor) handleGetDimensionValues(pluginCtx backend.PluginContext, parameters *simplejson.Json) ([]suggestData, error) {
region := parameters.Get("region").MustString()
namespace := parameters.Get("namespace").MustString()
metricName := parameters.Get("metricName").MustString()
dimensionKey := parameters.Get("dimensionKey").MustString()
dimensionsJson := parameters.Get("dimensions").MustMap()
func (e *cloudWatchExecutor) handleGetDimensionValues(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
region := parameters.Get("region")
namespace := parameters.Get("namespace")
metricName := parameters.Get("metricName")
dimensionKey := parameters.Get("dimensionKey")
dimensionsJson := parameters.Get("dimensions")
dimensionsValues := map[string]interface{}{}
err := json.Unmarshal([]byte(dimensionsJson), &dimensionsValues)
if err != nil {
return nil, fmt.Errorf("error unmarshaling dimension: %v", err)
}
var dimensions []*cloudwatch.DimensionFilter
addDimension := func(key string, value string) {
@ -329,8 +289,8 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(pluginCtx backend.PluginCo
}
dimensions = append(dimensions, filter)
}
for k, v := range dimensionsJson {
// due to legacy, value can be a string, a string slice or nil
for k, v := range dimensionsValues {
if vv, ok := v.(string); ok {
addDimension(k, vv)
} else if vv, ok := v.([]interface{}); ok {
@ -364,7 +324,7 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(pluginCtx backend.PluginCo
}
dupCheck[*dim.Value] = true
result = append(result, suggestData{Text: *dim.Value, Value: *dim.Value})
result = append(result, suggestData{Text: *dim.Value, Value: *dim.Value, Label: *dim.Value})
}
}
}
@ -376,9 +336,9 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(pluginCtx backend.PluginCo
return result, nil
}
func (e *cloudWatchExecutor) handleGetEbsVolumeIds(pluginCtx backend.PluginContext, parameters *simplejson.Json) ([]suggestData, error) {
region := parameters.Get("region").MustString()
instanceId := parameters.Get("instanceId").MustString()
func (e *cloudWatchExecutor) handleGetEbsVolumeIds(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
region := parameters.Get("region")
instanceId := parameters.Get("instanceId")
instanceIds := aws.StringSlice(parseMultiSelectValue(instanceId))
instances, err := e.ec2DescribeInstances(pluginCtx, region, nil, instanceIds)
@ -390,7 +350,7 @@ func (e *cloudWatchExecutor) handleGetEbsVolumeIds(pluginCtx backend.PluginConte
for _, reservation := range instances.Reservations {
for _, instance := range reservation.Instances {
for _, mapping := range instance.BlockDeviceMappings {
result = append(result, suggestData{Text: *mapping.Ebs.VolumeId, Value: *mapping.Ebs.VolumeId})
result = append(result, suggestData{Text: *mapping.Ebs.VolumeId, Value: *mapping.Ebs.VolumeId, Label: *mapping.Ebs.VolumeId})
}
}
}
@ -398,13 +358,19 @@ func (e *cloudWatchExecutor) handleGetEbsVolumeIds(pluginCtx backend.PluginConte
return result, nil
}
func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(pluginCtx backend.PluginContext, parameters *simplejson.Json) ([]suggestData, error) {
region := parameters.Get("region").MustString()
attributeName := parameters.Get("attributeName").MustString()
filterJson := parameters.Get("filters").MustMap()
func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
region := parameters.Get("region")
attributeName := parameters.Get("attributeName")
filterJson := parameters.Get("filters")
filterMap := map[string]interface{}{}
err := json.Unmarshal([]byte(filterJson), &filterMap)
if err != nil {
return nil, fmt.Errorf("error unmarshaling filter: %v", err)
}
var filters []*ec2.Filter
for k, v := range filterJson {
for k, v := range filterMap {
if vv, ok := v.([]interface{}); ok {
var values []*string
for _, vvv := range vv {
@ -466,7 +432,7 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(pluginCtx backend.Plu
}
dupCheck[data] = true
result = append(result, suggestData{Text: data, Value: data})
result = append(result, suggestData{Text: data, Value: data, Label: data})
}
}
@ -477,13 +443,19 @@ func (e *cloudWatchExecutor) handleGetEc2InstanceAttribute(pluginCtx backend.Plu
return result, nil
}
func (e *cloudWatchExecutor) handleGetResourceArns(pluginCtx backend.PluginContext, parameters *simplejson.Json) ([]suggestData, error) {
region := parameters.Get("region").MustString()
resourceType := parameters.Get("resourceType").MustString()
filterJson := parameters.Get("tags").MustMap()
func (e *cloudWatchExecutor) handleGetResourceArns(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error) {
region := parameters.Get("region")
resourceType := parameters.Get("resourceType")
tagsJson := parameters.Get("tags")
tagsMap := map[string]interface{}{}
err := json.Unmarshal([]byte(tagsJson), &tagsMap)
if err != nil {
return nil, fmt.Errorf("error unmarshaling filter: %v", err)
}
var filters []*resourcegroupstaggingapi.TagFilter
for k, v := range filterJson {
for k, v := range tagsMap {
if vv, ok := v.([]interface{}); ok {
var values []*string
for _, vvv := range vv {
@ -509,7 +481,7 @@ func (e *cloudWatchExecutor) handleGetResourceArns(pluginCtx backend.PluginConte
result := make([]suggestData, 0)
for _, resource := range resources.ResourceTagMappingList {
data := *resource.ResourceARN
result = append(result, suggestData{Text: data, Value: data})
result = append(result, suggestData{Text: data, Value: data, Label: data})
}
return result, nil

350
pkg/tsdb/cloudwatch/metric_find_query_test.go
View File

@ -1,8 +1,8 @@
package cloudwatch
import (
"context"
"encoding/json"
"net/url"
"testing"
"github.com/aws/aws-sdk-go/aws"
@ -54,40 +54,20 @@ func TestQuery_Metrics(t *testing.T) {
})
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
resp, err := executor.handleGetMetrics(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
}, url.Values{
"region": []string{"us-east-1"},
"namespace": []string{"custom"},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "metrics",
"region": "us-east-1",
"namespace": "custom"
}`),
},
},
})
)
require.NoError(t, err)
expFrame := data.NewFrame(
"",
data.NewField("text", nil, []string{"Test_MetricName"}),
data.NewField("value", nil, []string{"Test_MetricName"}),
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": 1,
},
expResponse := []suggestData{
{Text: "Test_MetricName", Value: "Test_MetricName", Label: "Test_MetricName"},
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
assert.Equal(t, expResponse, resp)
})
t.Run("Dimension keys for custom metrics", func(t *testing.T) {
@ -109,39 +89,20 @@ func TestQuery_Metrics(t *testing.T) {
})
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
resp, err := executor.handleGetDimensionKeys(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
}, url.Values{
"region": []string{"us-east-1"},
"namespace": []string{"custom"},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "dimension_keys",
"region": "us-east-1",
"namespace": "custom"
}`),
},
},
})
)
require.NoError(t, err)
expFrame := data.NewFrame(
"",
data.NewField("text", nil, []string{"Test_DimensionName"}),
data.NewField("value", nil, []string{"Test_DimensionName"}),
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": 1,
},
expResponse := []suggestData{
{Text: "Test_DimensionName", Value: "Test_DimensionName", Label: "Test_DimensionName"},
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
assert.Equal(t, expResponse, resp)
})
}
@ -168,21 +129,14 @@ func TestQuery_Regions(t *testing.T) {
})
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
resp, err := executor.handleGetRegions(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
}, url.Values{
"region": []string{"us-east-1"},
"namespace": []string{"custom"},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "regions",
"region": "us-east-1",
"namespace": "custom"
}`),
},
},
})
)
require.NoError(t, err)
expRegions := append(knownRegions, regionName)
@ -197,12 +151,11 @@ func TestQuery_Regions(t *testing.T) {
},
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
expResponse := []suggestData{}
for _, region := range expRegions {
expResponse = append(expResponse, suggestData{Text: region, Value: region, Label: region})
}
assert.Equal(t, expResponse, resp)
})
}
@ -242,44 +195,28 @@ func TestQuery_InstanceAttributes(t *testing.T) {
return datasourceInfo{}, nil
})
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "ec2_instance_attribute",
"region": "us-east-1",
"attributeName": "InstanceId",
"filters": {
"tag:Environment": ["production"]
}
}`),
},
},
})
filterMap := map[string][]string{
"tag:Environment": {"production"},
}
filterJson, err := json.Marshal(filterMap)
require.NoError(t, err)
expFrame := data.NewFrame(
"",
data.NewField("text", nil, []string{instanceID}),
data.NewField("value", nil, []string{instanceID}),
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.handleGetEc2InstanceAttribute(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
}, url.Values{
"region": []string{"us-east-1"},
"attributeName": []string{"InstanceId"},
"filters": []string{string(filterJson)},
},
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": 1,
},
}
require.NoError(t, err)
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
expResponse := []suggestData{
{Text: instanceID, Value: instanceID, Label: instanceID},
}
assert.Equal(t, expResponse, resp)
})
}
@ -342,41 +279,22 @@ func TestQuery_EBSVolumeIDs(t *testing.T) {
})
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
resp, err := executor.handleGetEbsVolumeIds(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
}, url.Values{
"region": []string{"us-east-1"},
"instanceId": []string{"{i-1, i-2, i-3}"},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "ebs_volume_ids",
"region": "us-east-1",
"instanceId": "{i-1, i-2, i-3}"
}`),
},
},
})
)
require.NoError(t, err)
expValues := []string{"vol-1-1", "vol-1-2", "vol-2-1", "vol-2-2", "vol-3-1", "vol-3-2"}
expFrame := data.NewFrame(
"",
data.NewField("text", nil, expValues),
data.NewField("value", nil, expValues),
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": 6,
},
expResponse := []suggestData{}
for _, value := range expValues {
expResponse = append(expResponse, suggestData{Text: value, Value: value, Label: value})
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
assert.Equal(t, expResponse, resp)
})
}
@ -420,48 +338,33 @@ func TestQuery_ResourceARNs(t *testing.T) {
return datasourceInfo{}, nil
})
tagMap := map[string][]string{
"Environment": {"production"},
}
tagJson, err := json.Marshal(tagMap)
require.NoError(t, err)
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
resp, err := executor.handleGetResourceArns(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
}, url.Values{
"region": []string{"us-east-1"},
"resourceType": []string{"ec2:instance"},
"tags": []string{string(tagJson)},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "resource_arns",
"region": "us-east-1",
"resourceType": "ec2:instance",
"tags": {
"Environment": ["production"]
}
}`),
},
},
})
)
require.NoError(t, err)
expValues := []string{
"arn:aws:ec2:us-east-1:123456789012:instance/i-12345678901234567",
"arn:aws:ec2:us-east-1:123456789012:instance/i-76543210987654321",
}
expFrame := data.NewFrame(
"",
data.NewField("text", nil, expValues),
data.NewField("value", nil, expValues),
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": 2,
},
expResponse := []suggestData{}
for _, value := range expValues {
expResponse = append(expResponse, suggestData{Text: value, Value: value, Label: value})
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
assert.Equal(t, expResponse, resp)
})
}
@ -472,20 +375,14 @@ func TestQuery_GetAllMetrics(t *testing.T) {
})
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
resp, err := executor.handleGetAllMetrics(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "all_metrics",
"region": "us-east-1"
}`),
},
url.Values{
"region": []string{"us-east-1"},
},
})
)
require.NoError(t, err)
metricCount := 0
@ -493,7 +390,7 @@ func TestQuery_GetAllMetrics(t *testing.T) {
metricCount += len(metrics)
}
assert.Equal(t, metricCount, resp.Responses[""].Frames[0].Fields[1].Len())
assert.Equal(t, metricCount, len(resp))
})
}
@ -527,46 +424,28 @@ func TestQuery_GetDimensionKeys(t *testing.T) {
})
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
resp, err := executor.handleGetDimensionKeys(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "dimension_keys",
"region": "us-east-1",
"namespace": "AWS/EC2",
"dimensionFilters": {
"InstanceId": "",
"AutoscalingGroup": []
}
}`),
},
url.Values{
"region": []string{"us-east-1"},
"namespace": []string{"AWS/EC2"},
"dimensionFilters": []string{`{
"InstanceId": "",
"AutoscalingGroup": []
}`},
},
})
)
require.NoError(t, err)
expValues := []string{"Dimension1", "Dimension2", "Dimension3"}
expFrame := data.NewFrame(
"",
data.NewField("text", nil, expValues),
data.NewField("value", nil, expValues),
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": len(expValues),
},
expResponse := []suggestData{}
for _, val := range expValues {
expResponse = append(expResponse, suggestData{val, val, val})
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
assert.Equal(t, expResponse, resp)
})
t.Run("should return hard coded metrics when no dimension filter is specified", func(t *testing.T) {
@ -575,42 +454,25 @@ func TestQuery_GetDimensionKeys(t *testing.T) {
})
executor := newExecutor(im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
resp, err := executor.handleGetDimensionKeys(
backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "dimension_keys",
"region": "us-east-1",
"namespace": "AWS/EC2",
"dimensionFilters": {}
}`),
},
url.Values{
"region": []string{"us-east-1"},
"namespace": []string{"AWS/EC2"},
"dimensionFilters": []string{`{}`},
},
})
)
require.NoError(t, err)
expValues := dimensionsMap["AWS/EC2"]
expFrame := data.NewFrame(
"",
data.NewField("text", nil, expValues),
data.NewField("value", nil, expValues),
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": len(expValues),
},
expResponse := []suggestData{}
for _, val := range expValues {
expResponse = append(expResponse, suggestData{val, val, val})
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
assert.Equal(t, expResponse, resp)
})
}
func Test_isCustomMetrics(t *testing.T) {

59
pkg/tsdb/cloudwatch/resource_handler.go Normal file
View File

@ -0,0 +1,59 @@
package cloudwatch
import (
"encoding/json"
"fmt"
"net/http"
"net/url"
"github.com/grafana/grafana-plugin-sdk-go/backend"
"github.com/grafana/grafana-plugin-sdk-go/backend/resource/httpadapter"
)
func (e *cloudWatchExecutor) newResourceMux() *http.ServeMux {
mux := http.NewServeMux()
mux.HandleFunc("/regions", handleResourceReq(e.handleGetRegions))
mux.HandleFunc("/namespaces", handleResourceReq(e.handleGetNamespaces))
mux.HandleFunc("/metrics", handleResourceReq(e.handleGetMetrics))
mux.HandleFunc("/all-metrics", handleResourceReq(e.handleGetAllMetrics))
mux.HandleFunc("/dimension-keys", handleResourceReq(e.handleGetDimensionKeys))
mux.HandleFunc("/dimension-values", handleResourceReq(e.handleGetDimensionValues))
mux.HandleFunc("/ebs-volume-ids", handleResourceReq(e.handleGetEbsVolumeIds))
mux.HandleFunc("/ec2-instance-attribute", handleResourceReq(e.handleGetEc2InstanceAttribute))
mux.HandleFunc("/resource-arns", handleResourceReq(e.handleGetResourceArns))
return mux
}
type handleFn func(pluginCtx backend.PluginContext, parameters url.Values) ([]suggestData, error)
func handleResourceReq(handleFunc handleFn) func(rw http.ResponseWriter, req *http.Request) {
return func(rw http.ResponseWriter, req *http.Request) {
ctx := req.Context()
pluginContext := httpadapter.PluginConfigFromContext(ctx)
err := req.ParseForm()
if err != nil {
writeResponse(rw, http.StatusBadRequest, fmt.Sprintf("unexpected error %v", err))
}
data, err := handleFunc(pluginContext, req.URL.Query())
if err != nil {
writeResponse(rw, http.StatusBadRequest, fmt.Sprintf("unexpected error %v", err))
}
body, err := json.Marshal(data)
if err != nil {
writeResponse(rw, http.StatusBadRequest, fmt.Sprintf("unexpected error %v", err))
}
rw.WriteHeader(http.StatusOK)
_, err = rw.Write(body)
if err != nil {
plog.Error("Unable to write HTTP response", "error", err)
}
}
}
func writeResponse(rw http.ResponseWriter, code int, msg string) {
rw.WriteHeader(code)
_, err := rw.Write([]byte(msg))
if err != nil {
plog.Error("Unable to write HTTP response", "error", err)
}
}

140
public/app/plugins/datasource/cloudwatch/datasource.ts
View File

@ -50,7 +50,6 @@ import {
StartQueryRequest,
TSDBResponse,
Dimensions,
MetricFindSuggestData,
CloudWatchLogsRequest,
} from './types';
import { CloudWatchLanguageProvider } from './language_provider';
@ -557,40 +556,8 @@ export class CloudWatchDatasource
);
}
transformSuggestDataFromDataframes(suggestData: TSDBResponse): MetricFindSuggestData[] {
const frames = toDataQueryResponse({ data: suggestData }).data as DataFrame[];
const table = toLegacyResponseData(frames[0]) as TableData;
return table.rows.map(([text, value]) => ({
text,
value,
label: value,
}));
}
doMetricQueryRequest(subtype: string, parameters: any): Promise<MetricFindSuggestData[]> {
const range = this.timeSrv.timeRange();
return lastValueFrom(
this.awsRequest(DS_QUERY_ENDPOINT, {
from: range.from.valueOf().toString(),
to: range.to.valueOf().toString(),
queries: [
{
refId: 'metricFindQuery',
intervalMs: 1, // dummy
maxDataPoints: 1, // dummy
datasource: this.getRef(),
type: 'metricFindQuery',
subtype: subtype,
...parameters,
},
],
}).pipe(
map((r) => {
return this.transformSuggestDataFromDataframes(r);
})
)
);
doMetricResourceRequest(subtype: string, parameters?: any): Promise<Array<{ text: any; label: any; value: any }>> {
return this.getResource(subtype, parameters);
}
makeLogActionRequest(
@ -677,14 +644,14 @@ export class CloudWatchDatasource
}
getRegions(): Promise<Array<{ label: string; value: string; text: string }>> {
return this.doMetricQueryRequest('regions', null).then((regions: any) => [
return this.doMetricResourceRequest('regions').then((regions: any) => [
{ label: 'default', value: 'default', text: 'default' },
...regions,
]);
}
getNamespaces() {
return this.doMetricQueryRequest('namespaces', null);
return this.doMetricResourceRequest('namespaces');
}
async getMetrics(namespace: string | undefined, region?: string) {
@ -692,14 +659,14 @@ export class CloudWatchDatasource
return [];
}
return this.doMetricQueryRequest('metrics', {
return this.doMetricResourceRequest('metrics', {
region: this.templateSrv.replace(this.getActualRegion(region)),
namespace: this.templateSrv.replace(namespace),
});
}
async getAllMetrics(region: string): Promise<Array<{ metricName: string; namespace: string }>> {
const values = await this.doMetricQueryRequest('all_metrics', {
const values = await this.doMetricResourceRequest('all-metrics', {
region: this.templateSrv.replace(this.getActualRegion(region)),
});
@ -716,10 +683,10 @@ export class CloudWatchDatasource
return [];
}
return this.doMetricQueryRequest('dimension_keys', {
return this.doMetricResourceRequest('dimension-keys', {
region: this.templateSrv.replace(this.getActualRegion(region)),
namespace: this.templateSrv.replace(namespace),
dimensionFilters: this.convertDimensionFormat(dimensionFilters, {}),
dimensionFilters: JSON.stringify(this.convertDimensionFormat(dimensionFilters, {})),
metricName,
});
}
@ -735,117 +702,40 @@ export class CloudWatchDatasource
return [];
}
const values = await this.doMetricQueryRequest('dimension_values', {
const values = await this.doMetricResourceRequest('dimension-values', {
region: this.templateSrv.replace(this.getActualRegion(region)),
namespace: this.templateSrv.replace(namespace),
metricName: this.templateSrv.replace(metricName.trim()),
dimensionKey: this.templateSrv.replace(dimensionKey),
dimensions: this.convertDimensionFormat(filterDimensions, {}),
dimensions: JSON.stringify(this.convertDimensionFormat(filterDimensions, {})),
});
return values;
}
getEbsVolumeIds(region: string, instanceId: string) {
return this.doMetricQueryRequest('ebs_volume_ids', {
return this.doMetricResourceRequest('ebs-volume-ids', {
region: this.templateSrv.replace(this.getActualRegion(region)),
instanceId: this.templateSrv.replace(instanceId),
});
}
getEc2InstanceAttribute(region: string, attributeName: string, filters: any) {
return this.doMetricQueryRequest('ec2_instance_attribute', {
return this.doMetricResourceRequest('ec2-instance-attribute', {
region: this.templateSrv.replace(this.getActualRegion(region)),
attributeName: this.templateSrv.replace(attributeName),
filters: filters,
filters: JSON.stringify(filters),
});
}
getResourceARNs(region: string, resourceType: string, tags: any) {
return this.doMetricQueryRequest('resource_arns', {
return this.doMetricResourceRequest('resource-arns', {
region: this.templateSrv.replace(this.getActualRegion(region)),
resourceType: this.templateSrv.replace(resourceType),
tags: tags,
tags: JSON.stringify(tags),
});
}
async metricFindQuery(query: string) {
let region;
let namespace;
let metricName;
let filterJson;
const regionQuery = query.match(/^regions\(\)/);
if (regionQuery) {
return this.getRegions();
}
const namespaceQuery = query.match(/^namespaces\(\)/);
if (namespaceQuery) {
return this.getNamespaces();
}
const metricNameQuery = query.match(/^metrics\(([^\)]+?)(,\s?([^,]+?))?\)/);
if (metricNameQuery) {
namespace = metricNameQuery[1];
region = metricNameQuery[3];
return this.getMetrics(namespace, region);
}
const dimensionKeysQuery = query.match(/^dimension_keys\(([^\)]+?)(,\s?([^,]+?))?\)/);
if (dimensionKeysQuery) {
namespace = dimensionKeysQuery[1];
region = dimensionKeysQuery[3];
return this.getDimensionKeys(namespace, region);
}
const dimensionValuesQuery = query.match(
/^dimension_values\(([^,]+?),\s?([^,]+?),\s?([^,]+?),\s?([^,]+?)(,\s?(.+))?\)/
);
if (dimensionValuesQuery) {
region = dimensionValuesQuery[1];
namespace = dimensionValuesQuery[2];
metricName = dimensionValuesQuery[3];
const dimensionKey = dimensionValuesQuery[4];
filterJson = {};
if (dimensionValuesQuery[6]) {
filterJson = JSON.parse(this.templateSrv.replace(dimensionValuesQuery[6]));
}
return this.getDimensionValues(region, namespace, metricName, dimensionKey, filterJson);
}
const ebsVolumeIdsQuery = query.match(/^ebs_volume_ids\(([^,]+?),\s?([^,]+?)\)/);
if (ebsVolumeIdsQuery) {
region = ebsVolumeIdsQuery[1];
const instanceId = ebsVolumeIdsQuery[2];
return this.getEbsVolumeIds(region, instanceId);
}
const ec2InstanceAttributeQuery = query.match(/^ec2_instance_attribute\(([^,]+?),\s?([^,]+?),\s?(.+?)\)/);
if (ec2InstanceAttributeQuery) {
region = ec2InstanceAttributeQuery[1];
const targetAttributeName = ec2InstanceAttributeQuery[2];
filterJson = JSON.parse(this.templateSrv.replace(ec2InstanceAttributeQuery[3]));
return this.getEc2InstanceAttribute(region, targetAttributeName, filterJson);
}
const resourceARNsQuery = query.match(/^resource_arns\(([^,]+?),\s?([^,]+?),\s?(.+?)\)/);
if (resourceARNsQuery) {
region = resourceARNsQuery[1];
const resourceType = resourceARNsQuery[2];
const tagsJSON = JSON.parse(this.templateSrv.replace(resourceARNsQuery[3]));
return this.getResourceARNs(region, resourceType, tagsJSON);
}
const statsQuery = query.match(/^statistics\(\)/);
if (statsQuery) {
return this.standardStatistics.map((s: string) => ({ value: s, label: s, text: s }));
}
return Promise.resolve([]);
}
annotationQuery(options: any) {
const annotation = options.annotation;
const statistic = this.templateSrv.replace(annotation.statistic);

30
public/app/plugins/datasource/cloudwatch/specs/datasource.test.ts
View File

@ -506,36 +506,6 @@ describe('CloudWatchDatasource', () => {
});
});
});
describe('when regions query is used', () => {
describe('and region param is left out', () => {
it('should use the default region', async () => {
const { ds, instanceSettings } = getTestContext();
ds.doMetricQueryRequest = jest.fn().mockResolvedValue([]);
await ds.metricFindQuery('metrics(testNamespace)');
expect(ds.doMetricQueryRequest).toHaveBeenCalledWith('metrics', {
namespace: 'testNamespace',
region: instanceSettings.jsonData.defaultRegion,
});
});
});
describe('and region param is defined by user', () => {
it('should use the user defined region', async () => {
const { ds } = getTestContext();
ds.doMetricQueryRequest = jest.fn().mockResolvedValue([]);
await ds.metricFindQuery('metrics(testNamespace2, custom-region)');
expect(ds.doMetricQueryRequest).toHaveBeenCalledWith('metrics', {
namespace: 'testNamespace2',
region: 'custom-region',
});
});
});
});
});
describe('When query region is "default"', () => {

6
public/app/plugins/datasource/cloudwatch/types.ts
View File

@ -367,9 +367,3 @@ export interface MetricQuery {
maxDataPoints?: number;
intervalMs?: number;
}
export interface MetricFindSuggestData {
text: string;
label: string;
value: string;
}