Cloudwatch: Enable dimension filtering when loading dimension values (#41566)

* fix dimension filter

* refactor tests

* add comments

* fix typo
This commit is contained in:
Erik Sundell 2021-11-11 16:48:35 +01:00 committed by GitHub
parent 0f36152127
commit eb40723bcb
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 418 additions and 54 deletions

View File

@ -276,6 +276,8 @@ func (e *cloudWatchExecutor) executeMetricFindQuery(ctx context.Context, model *
data, err = e.handleGetNamespaces(ctx, model, pluginCtx)
case "metrics":
data, err = e.handleGetMetrics(ctx, model, pluginCtx)
case "all_metrics":
data, err = e.handleGetAllMetrics(ctx, model, pluginCtx)
case "dimension_keys":
data, err = e.handleGetDimensions(ctx, model, pluginCtx)
case "dimension_values":
@ -434,15 +436,90 @@ func (e *cloudWatchExecutor) handleGetMetrics(ctx context.Context, parameters *s
return result, nil
}
// handleGetAllMetrics returns a slice of suggestData structs with metric and its namespace
func (e *cloudWatchExecutor) handleGetAllMetrics(ctx context.Context, parameters *simplejson.Json, pluginCtx backend.PluginContext) ([]suggestData, error) {
result := make([]suggestData, 0)
for namespace, metrics := range metricsMap {
for _, metric := range metrics {
result = append(result, suggestData{Text: namespace, Value: metric})
}
}
return result, nil
}
// handleGetDimensions returns a slice of suggestData structs with dimension keys.
// If a dimension filters parameter is specified, a new api call to list metrics will be issued to load dimension keys for the given filter.
// If no dimension filter is specified, dimension keys will be retrieved from the hard coded map in this file.
func (e *cloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters *simplejson.Json, pluginCtx backend.PluginContext) ([]suggestData, error) {
region := parameters.Get("region").MustString()
namespace := parameters.Get("namespace").MustString()
metricName := parameters.Get("metricName").MustString("")
dimensionFilters := parameters.Get("dimensionFilters").MustMap()
var dimensionValues []string
if !isCustomMetrics(namespace) {
var exists bool
if dimensionValues, exists = dimensionsMap[namespace]; !exists {
return nil, fmt.Errorf("unable to find dimension %q", namespace)
if len(dimensionFilters) != 0 {
var dimensions []*cloudwatch.DimensionFilter
addDimension := func(key string, value string) {
filter := &cloudwatch.DimensionFilter{
Name: aws.String(key),
}
// if value is not specified or a wildcard is used, simply don't use the value field
if value != "" && value != "*" {
filter.Value = aws.String(value)
}
dimensions = append(dimensions, filter)
}
for k, v := range dimensionFilters {
// due to legacy, value can be a string, a string slice or nil
if vv, ok := v.(string); ok {
addDimension(k, vv)
} else if vv, ok := v.([]interface{}); ok {
for _, v := range vv {
addDimension(k, v.(string))
}
} else if v == nil {
addDimension(k, "")
}
}
input := &cloudwatch.ListMetricsInput{
Namespace: aws.String(namespace),
Dimensions: dimensions,
}
if metricName != "" {
input.MetricName = aws.String(metricName)
}
metrics, err := e.listMetrics(region, input, pluginCtx)
if err != nil {
return nil, errutil.Wrap("unable to call AWS API", err)
}
dupCheck := make(map[string]bool)
for _, metric := range metrics {
for _, dim := range metric.Dimensions {
if _, exists := dupCheck[*dim.Name]; exists {
continue
}
// keys in the dimension filter should not be included
if _, ok := dimensionFilters[*dim.Name]; ok {
continue
}
dupCheck[*dim.Name] = true
dimensionValues = append(dimensionValues, *dim.Name)
}
}
} else {
var exists bool
if dimensionValues, exists = dimensionsMap[namespace]; !exists {
return nil, fmt.Errorf("unable to find dimension %q", namespace)
}
}
} else {
var err error
@ -460,6 +537,8 @@ func (e *cloudWatchExecutor) handleGetDimensions(ctx context.Context, parameters
return result, nil
}
// handleGetDimensionValues returns a slice of suggestData structs with dimension values.
// A call to the list metrics api is issued to retrieve the dimension values. All parameters are used as input args to the list metrics call.
func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, parameters *simplejson.Json, pluginCtx backend.PluginContext) ([]suggestData, error) {
region := parameters.Get("region").MustString()
namespace := parameters.Get("namespace").MustString()
@ -468,19 +547,26 @@ func (e *cloudWatchExecutor) handleGetDimensionValues(ctx context.Context, param
dimensionsJson := parameters.Get("dimensions").MustMap()
var dimensions []*cloudwatch.DimensionFilter
addDimension := func(key string, value string) {
filter := &cloudwatch.DimensionFilter{
Name: aws.String(key),
}
// if value is not specified or a wildcard is used, simply don't use the value field
if value != "" && value != "*" {
filter.Value = aws.String(value)
}
dimensions = append(dimensions, filter)
}
for k, v := range dimensionsJson {
// due to legacy, value can be a string, a string slice or nil
if vv, ok := v.(string); ok {
dimensions = append(dimensions, &cloudwatch.DimensionFilter{
Name: aws.String(k),
Value: aws.String(vv),
})
addDimension(k, vv)
} else if vv, ok := v.([]interface{}); ok {
for _, v := range vv {
dimensions = append(dimensions, &cloudwatch.DimensionFilter{
Name: aws.String(k),
Value: aws.String(v.(string)),
})
addDimension(k, v.(string))
}
} else if v == nil {
addDimension(k, "")
}
}

View File

@ -465,6 +465,154 @@ func TestQuery_ResourceARNs(t *testing.T) {
})
}
func TestQuery_GetAllMetrics(t *testing.T) {
t.Run("all metrics in all namespaces are being returned", func(t *testing.T) {
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return datasourceInfo{}, nil
})
executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "all_metrics",
"region": "us-east-1"
}`),
},
},
})
require.NoError(t, err)
metricCount := 0
for _, metrics := range metricsMap {
metricCount += len(metrics)
}
assert.Equal(t, metricCount, resp.Responses[""].Frames[0].Fields[1].Len())
})
}
func TestQuery_GetDimensionKeys(t *testing.T) {
origNewCWClient := NewCWClient
t.Cleanup(func() {
NewCWClient = origNewCWClient
})
var client FakeCWClient
NewCWClient = func(sess *session.Session) cloudwatchiface.CloudWatchAPI {
return client
}
metrics := []*cloudwatch.Metric{
{MetricName: aws.String("Test_MetricName1"), Dimensions: []*cloudwatch.Dimension{
{Name: aws.String("Dimension1"), Value: aws.String("Dimension1")},
{Name: aws.String("Dimension2"), Value: aws.String("Dimension2")},
}},
{MetricName: aws.String("Test_MetricName2"), Dimensions: []*cloudwatch.Dimension{
{Name: aws.String("Dimension2"), Value: aws.String("Dimension2")},
{Name: aws.String("Dimension3"), Value: aws.String("Dimension3")},
}},
}
t.Run("should fetch dimension keys from list metrics api and return unique dimensions when a dimension filter is specified", func(t *testing.T) {
client = FakeCWClient{Metrics: metrics, MetricsPerPage: 2}
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return datasourceInfo{}, nil
})
executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "dimension_keys",
"region": "us-east-1",
"namespace": "AWS/EC2",
"dimensionFilters": {
"InstanceId": "",
"AutoscalingGroup": []
}
}`),
},
},
})
require.NoError(t, err)
expValues := []string{"Dimension1", "Dimension2", "Dimension3"}
expFrame := data.NewFrame(
"",
data.NewField("text", nil, expValues),
data.NewField("value", nil, expValues),
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": len(expValues),
},
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
})
t.Run("should return hard coded metrics when no dimension filter is specified", func(t *testing.T) {
im := datasource.NewInstanceManager(func(s backend.DataSourceInstanceSettings) (instancemgmt.Instance, error) {
return datasourceInfo{}, nil
})
executor := newExecutor(nil, im, newTestConfig(), fakeSessionCache{})
resp, err := executor.QueryData(context.Background(), &backend.QueryDataRequest{
PluginContext: backend.PluginContext{
DataSourceInstanceSettings: &backend.DataSourceInstanceSettings{},
},
Queries: []backend.DataQuery{
{
JSON: json.RawMessage(`{
"type": "metricFindQuery",
"subtype": "dimension_keys",
"region": "us-east-1",
"namespace": "AWS/EC2",
"dimensionFilters": {}
}`),
},
},
})
require.NoError(t, err)
expValues := dimensionsMap["AWS/EC2"]
expFrame := data.NewFrame(
"",
data.NewField("text", nil, expValues),
data.NewField("value", nil, expValues),
)
expFrame.Meta = &data.FrameMeta{
Custom: map[string]interface{}{
"rowCount": len(expValues),
},
}
assert.Equal(t, &backend.QueryDataResponse{Responses: backend.Responses{
"": {
Frames: data.Frames{expFrame},
},
},
}, resp)
})
}
func Test_isCustomMetrics(t *testing.T) {
metricsMap = map[string][]string{
"AWS/EC2": {"ExampleMetric"},

View File

@ -0,0 +1,121 @@
import { dateTime } from '@grafana/data';
import { setBackendSrv } from '@grafana/runtime';
import { TemplateSrvMock } from '../../../../features/templating/template_srv.mock';
import { initialCustomVariableModelState } from 'app/features/variables/custom/reducer';
import { CustomVariableModel } from 'app/features/variables/types';
import { of } from 'rxjs';
import { CloudWatchDatasource } from '../datasource';
import { TemplateSrv } from 'app/features/templating/template_srv';
export function setupMockedDataSource({ data = [], variables }: { data?: any; variables?: any } = {}) {
let templateService = new TemplateSrvMock({
region: 'templatedRegion',
fields: 'templatedField',
group: 'templatedGroup',
}) as any;
if (variables) {
templateService = new TemplateSrv();
templateService.init(variables);
}
const datasource = new CloudWatchDatasource(
{
jsonData: { defaultRegion: 'us-west-1', tracingDatasourceUid: 'xray' },
} as any,
templateService,
{
timeRange() {
const time = dateTime('2021-01-01T01:00:00Z');
const range = {
from: time.subtract(6, 'hour'),
to: time,
};
return {
...range,
raw: range,
};
},
} as any
);
const fetchMock = jest.fn().mockReturnValue(of({ data }));
setBackendSrv({ fetch: fetchMock } as any);
return { datasource, fetchMock };
}
export const metricVariable: CustomVariableModel = {
...initialCustomVariableModelState,
id: 'metric',
name: 'metric',
current: { value: 'CPUUtilization', text: 'CPUUtilizationEC2', selected: true },
options: [
{ value: 'DroppedBytes', text: 'DroppedBytes', selected: false },
{ value: 'CPUUtilization', text: 'CPUUtilization', selected: false },
],
multi: false,
};
export const namespaceVariable: CustomVariableModel = {
...initialCustomVariableModelState,
id: 'namespace',
name: 'namespace',
query: 'namespaces()',
current: { value: 'AWS/EC2', text: 'AWS/EC2', selected: true },
options: [
{ value: 'AWS/Redshift', text: 'AWS/Redshift', selected: false },
{ value: 'AWS/EC2', text: 'AWS/EC2', selected: false },
{ value: 'AWS/MQ', text: 'AWS/MQ', selected: false },
],
multi: false,
};
export const labelsVariable: CustomVariableModel = {
...initialCustomVariableModelState,
id: 'labels',
name: 'labels',
current: {
value: ['InstanceId', 'InstanceType'],
text: ['InstanceId', 'InstanceType'].toString(),
selected: true,
},
options: [
{ value: 'InstanceId', text: 'InstanceId', selected: false },
{ value: 'InstanceType', text: 'InstanceType', selected: false },
],
multi: true,
};
export const limitVariable: CustomVariableModel = {
...initialCustomVariableModelState,
id: 'limit',
name: 'limit',
current: {
value: '100',
text: '100',
selected: true,
},
options: [
{ value: '10', text: '10', selected: false },
{ value: '100', text: '100', selected: false },
{ value: '1000', text: '1000', selected: false },
],
multi: false,
};
export const aggregationvariable: CustomVariableModel = {
...initialCustomVariableModelState,
id: 'aggregation',
name: 'aggregation',
current: {
value: 'AVG',
text: 'AVG',
selected: true,
},
options: [
{ value: 'AVG', text: 'AVG', selected: false },
{ value: 'SUM', text: 'SUM', selected: false },
{ value: 'MIN', text: 'MIN', selected: false },
],
multi: false,
};

View File

@ -1,16 +1,15 @@
import { lastValueFrom, of } from 'rxjs';
import { setBackendSrv, setDataSourceSrv } from '@grafana/runtime';
import { setDataSourceSrv } from '@grafana/runtime';
import { ArrayVector, DataFrame, dataFrameToJSON, dateTime, Field, MutableDataFrame } from '@grafana/data';
import { CloudWatchDatasource } from './datasource';
import { toArray } from 'rxjs/operators';
import { setupMockedDataSource } from './__mocks__/CloudWatchDataSource';
import { CloudWatchLogsQueryStatus } from './types';
import { TemplateSrvMock } from '../../../features/templating/template_srv.mock';
describe('datasource', () => {
describe('query', () => {
it('should return error if log query and log groups is not specified', async () => {
const { datasource } = setup();
const { datasource } = setupMockedDataSource();
const observable = datasource.query({ targets: [{ queryMode: 'Logs' as 'Logs' }] } as any);
await expect(observable).toEmitValuesWith((received) => {
@ -20,7 +19,7 @@ describe('datasource', () => {
});
it('should return empty response if queries are hidden', async () => {
const { datasource } = setup();
const { datasource } = setupMockedDataSource();
const observable = datasource.query({ targets: [{ queryMode: 'Logs' as 'Logs', hide: true }] } as any);
await expect(observable).toEmitValuesWith((received) => {
@ -30,7 +29,7 @@ describe('datasource', () => {
});
it('should interpolate variables in the query', async () => {
const { datasource, fetchMock } = setup();
const { datasource, fetchMock } = setupMockedDataSource();
datasource.query({
targets: [
{
@ -86,7 +85,7 @@ describe('datasource', () => {
describe('performTimeSeriesQuery', () => {
it('should return the same length of data as result', async () => {
const { datasource } = setup({
const { datasource } = setupMockedDataSource({
data: {
results: {
a: { refId: 'a', series: [{ name: 'cpu', points: [1, 1] }], meta: {} },
@ -114,7 +113,7 @@ describe('datasource', () => {
describe('describeLogGroup', () => {
it('replaces region correctly in the query', async () => {
const { datasource, fetchMock } = setup();
const { datasource, fetchMock } = setupMockedDataSource();
await datasource.describeLogGroups({ region: 'default' });
expect(fetchMock.mock.calls[0][0].data.queries[0].region).toBe('us-west-1');
@ -124,37 +123,12 @@ describe('datasource', () => {
});
});
function setup({ data = [] }: { data?: any } = {}) {
const datasource = new CloudWatchDatasource(
{ jsonData: { defaultRegion: 'us-west-1', tracingDatasourceUid: 'xray' } } as any,
new TemplateSrvMock({ region: 'templatedRegion', fields: 'templatedField', group: 'templatedGroup' }) as any,
{
timeRange() {
const time = dateTime('2021-01-01T01:00:00Z');
const range = {
from: time.subtract(6, 'hour'),
to: time,
};
return {
...range,
raw: range,
};
},
} as any
);
const fetchMock = jest.fn().mockReturnValue(of({ data }));
setBackendSrv({ fetch: fetchMock } as any);
return { datasource, fetchMock };
}
function setupForLogs() {
function envelope(frame: DataFrame) {
return { data: { results: { a: { refId: 'a', frames: [dataFrameToJSON(frame)] } } } };
}
const { datasource, fetchMock } = setup();
const { datasource, fetchMock } = setupMockedDataSource();
const startQueryFrame = new MutableDataFrame({ fields: [{ name: 'queryId', values: ['queryid'] }] });
fetchMock.mockReturnValueOnce(of(envelope(startQueryFrame)));

View File

@ -46,6 +46,8 @@ import {
MetricRequest,
StartQueryRequest,
TSDBResponse,
Dimensions,
MetricFindSuggestData,
} from './types';
import { CloudWatchLanguageProvider } from './language_provider';
import { VariableWithMultiSupport } from 'app/features/variables/types';
@ -476,7 +478,12 @@ export class CloudWatchDatasource
(refId && !failedRedIds.includes(refId)) || res.includes(region) ? res : [...res, region],
[]
) as string[];
regionsAffected.forEach((region) => this.debouncedAlert(this.datasourceName, this.getActualRegion(region)));
regionsAffected.forEach((region) => {
const actualRegion = this.getActualRegion(region);
if (actualRegion) {
this.debouncedAlert(this.datasourceName, actualRegion);
}
});
}
return throwError(() => err);
@ -484,7 +491,7 @@ export class CloudWatchDatasource
);
}
transformSuggestDataFromDataframes(suggestData: TSDBResponse): Array<{ text: any; label: any; value: any }> {
transformSuggestDataFromDataframes(suggestData: TSDBResponse): MetricFindSuggestData[] {
const frames = toDataQueryResponse({ data: suggestData }).data as DataFrame[];
const table = toLegacyResponseData(frames[0]) as TableData;
@ -495,7 +502,7 @@ export class CloudWatchDatasource
}));
}
doMetricQueryRequest(subtype: string, parameters: any): Promise<Array<{ text: any; label: any; value: any }>> {
doMetricQueryRequest(subtype: string, parameters: any): Promise<MetricFindSuggestData[]> {
const range = this.timeSrv.timeRange();
return lastValueFrom(
this.awsRequest(DS_QUERY_ENDPOINT, {
@ -604,7 +611,7 @@ export class CloudWatchDatasource
return this.doMetricQueryRequest('namespaces', null);
}
async getMetrics(namespace: string, region?: string) {
async getMetrics(namespace: string | undefined, region?: string) {
if (!namespace) {
return [];
}
@ -615,7 +622,20 @@ export class CloudWatchDatasource
});
}
async getDimensionKeys(namespace: string, region: string) {
async getAllMetrics(region: string): Promise<Array<{ metricName: string; namespace: string }>> {
const values = await this.doMetricQueryRequest('all_metrics', {
region: this.templateSrv.replace(this.getActualRegion(region)),
});
return values.map((v) => ({ metricName: v.label, namespace: v.text }));
}
async getDimensionKeys(
namespace: string | undefined,
region: string,
dimensionFilters: Dimensions = {},
metricName = ''
) {
if (!namespace) {
return [];
}
@ -623,13 +643,15 @@ export class CloudWatchDatasource
return this.doMetricQueryRequest('dimension_keys', {
region: this.templateSrv.replace(this.getActualRegion(region)),
namespace: this.templateSrv.replace(namespace),
dimensionFilters: this.convertDimensionFormat(dimensionFilters, {}),
metricName,
});
}
async getDimensionValues(
region: string,
namespace: string,
metricName: string,
namespace: string | undefined,
metricName: string | undefined,
dimensionKey: string,
filterDimensions: {}
) {
@ -813,7 +835,7 @@ export class CloudWatchDatasource
const dimensions = {};
try {
await this.getDimensionValues(region, namespace, metricName, 'ServiceName', dimensions);
await this.getDimensionValues(region ?? '', namespace, metricName, 'ServiceName', dimensions);
return {
status: 'success',
message: 'Data source is working',
@ -858,7 +880,7 @@ export class CloudWatchDatasource
return Math.round(date.valueOf() / 1000);
}
convertDimensionFormat(dimensions: { [key: string]: string | string[] }, scopedVars: ScopedVars) {
convertDimensionFormat(dimensions: Dimensions, scopedVars: ScopedVars) {
return Object.entries(dimensions).reduce((result, [key, value]) => {
key = this.replace(key, scopedVars, true, 'dimension keys');
@ -866,6 +888,10 @@ export class CloudWatchDatasource
return { ...result, [key]: value };
}
if (!value) {
return { ...result, [key]: null };
}
const valueVar = this.templateSrv
.getVariables()
.find(({ name }) => name === this.templateSrv.getVariableName(value));

View File

@ -1,5 +1,8 @@
import { DataQuery, DataSourceRef, SelectableValue } from '@grafana/data';
import { AwsAuthDataSourceSecureJsonData, AwsAuthDataSourceJsonData } from '@grafana/aws-sdk';
export interface Dimensions {
[key: string]: string | string[];
}
export interface CloudWatchMetricsQuery extends DataQuery {
queryMode?: 'Metrics';
@ -325,3 +328,9 @@ export interface ExecutedQueryPreview {
executedQuery: string;
period: string;
}
export interface MetricFindSuggestData {
text: string;
label: string;
value: string;
}