IntenseWebs/grafana
3
0
Fork 0
mirror of https://github.com/grafana/grafana.git synced 2025-02-25 18:55:37 -06:00
Code Issues Packages Projects Releases Wiki Activity

CloudMonitoring: Support request cancellation properly (#28847)

Browse Source
This commit is contained in:
Hugo Häggmark 2020-11-19 14:09:08 +01:00 committed by GitHub
parent 294770f411
commit f9281742d7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 335 additions and 309 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

74
public/app/plugins/datasource/cloud-monitoring/api.test.ts
View File

@ -1,6 +1,8 @@
import { of } from 'rxjs';
import Api from './api';
import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
import { SelectableValue } from '@grafana/data';
import { createFetchResponse } from 'test/helpers/createFetchResponse';
jest.mock('@grafana/runtime', () => ({
...((jest.requireActual('@grafana/runtime') as unknown) as object),
@ -12,58 +14,60 @@ const response = [
{ label: 'test2', value: 'test2' },
];
describe('api', () => {
const datasourceRequestMock = jest.spyOn(backendSrv, 'datasourceRequest');
beforeEach(() => {
datasourceRequestMock.mockImplementation((options: any) => {
type Args = { path?: string; options?: any; response?: any; cache?: any };
async function getTestContext({ path = 'some-resource', options = {}, response = {}, cache }: Args = {}) {
jest.clearAllMocks();
const fetchMock = jest.spyOn(backendSrv, 'fetch');
fetchMock.mockImplementation((options: any) => {
const data = { [options.url.match(/([^\/]*)\/*$/)[1]]: response };
return Promise.resolve({ data, status: 200 });
});
return of(createFetchResponse(data));
});
const api = new Api('/cloudmonitoring/');
if (cache) {
api.cache[path] = cache;
}
const res = await api.get(path, options);
return { res, api, fetchMock };
}
describe('api', () => {
describe('when resource was cached', () => {
let api: Api;
let res: Array<SelectableValue<string>>;
beforeEach(async () => {
api = new Api('/cloudmonitoring/');
api.cache['some-resource'] = response;
res = await api.get('some-resource');
});
it('should return cached value and not load from source', async () => {
const path = 'some-resource';
const { res, api, fetchMock } = await getTestContext({ path, cache: response });
it('should return cached value and not load from source', () => {
expect(res).toEqual(response);
expect(api.cache['some-resource']).toEqual(response);
expect(datasourceRequestMock).not.toHaveBeenCalled();
expect(api.cache[path]).toEqual(response);
expect(fetchMock).not.toHaveBeenCalled();
});
});
describe('when resource was not cached', () => {
let api: Api;
let res: Array<SelectableValue<string>>;
beforeEach(async () => {
api = new Api('/cloudmonitoring/');
res = await api.get('some-resource');
});
it('should return from source and not from cache', async () => {
const path = 'some-resource';
const { res, api, fetchMock } = await getTestContext({ path, response });
it('should return cached value and not load from source', () => {
expect(res).toEqual(response);
expect(api.cache['some-resource']).toEqual(response);
expect(datasourceRequestMock).toHaveBeenCalled();
expect(api.cache[path]).toEqual(response);
expect(fetchMock).toHaveBeenCalled();
});
});
describe('when cache should be bypassed', () => {
let api: Api;
let res: Array<SelectableValue<string>>;
beforeEach(async () => {
api = new Api('/cloudmonitoring/');
api.cache['some-resource'] = response;
res = await api.get('some-resource', { useCache: false });
});
it('should return from source and not from cache', async () => {
const options = { useCache: false };
const path = 'some-resource';
const { res, fetchMock } = await getTestContext({ path, response, cache: response, options });
it('should return cached value and not load from source', () => {
expect(res).toEqual(response);
expect(datasourceRequestMock).toHaveBeenCalled();
expect(fetchMock).toHaveBeenCalled();
});
});
});

50
public/app/plugins/datasource/cloud-monitoring/api.ts
View File

@ -1,11 +1,17 @@
import { Observable, of } from 'rxjs';
import { catchError, map } from 'rxjs/operators';
import { SelectableValue } from '@grafana/data';
import { FetchResponse, getBackendSrv } from '@grafana/runtime';
import appEvents from 'app/core/app_events';
import { CoreEvents } from 'app/types';
import { SelectableValue } from '@grafana/data';
import { getBackendSrv } from '@grafana/runtime';
import { formatCloudMonitoringError } from './functions';
import { MetricDescriptor } from './types';
export interface PostResponse {
results: Record<string, any>;
}
interface Options {
responseMap?: (res: any) => SelectableValue<string> | MetricDescriptor;
baseUrl?: string;
@ -25,19 +31,20 @@ export default class Api {
};
}
async get(path: string, options?: Options): Promise<Array<SelectableValue<string>> | MetricDescriptor[]> {
try {
get(path: string, options?: Options): Promise<Array<SelectableValue<string>> | MetricDescriptor[]> {
const { useCache, responseMap, baseUrl } = { ...this.defaultOptions, ...options };
if (useCache && this.cache[path]) {
return this.cache[path];
return Promise.resolve(this.cache[path]);
}
const response = await getBackendSrv().datasourceRequest({
return getBackendSrv()
.fetch<Record<string, any>>({
url: baseUrl + path,
method: 'GET',
});
})
.pipe(
map(response => {
const responsePropName = path.match(/([^\/]*)\/*$/)![1];
let res = [];
if (response && response.data && response.data[responsePropName]) {
@ -49,24 +56,31 @@ export default class Api {
}
return res;
} catch (error) {
appEvents.emit(CoreEvents.dsRequestError, { error: { data: { error: formatCloudMonitoringError(error) } } });
return [];
}
}),
catchError(error => {
appEvents.emit(CoreEvents.dsRequestError, {
error: { data: { error: formatCloudMonitoringError(error) } },
});
return of([]);
})
)
.toPromise();
}
async post(data: { [key: string]: any }) {
return getBackendSrv().datasourceRequest({
post(data: Record<string, any>): Observable<FetchResponse<PostResponse>> {
return getBackendSrv().fetch<PostResponse>({
url: '/api/tsdb/query',
method: 'POST',
data,
});
}
async test(projectName: string) {
return getBackendSrv().datasourceRequest({
test(projectName: string) {
return getBackendSrv()
.fetch<any>({
url: `${this.baseUrl}${projectName}/metricDescriptors`,
method: 'GET',
});
})
.toPromise();
}
}

89
public/app/plugins/datasource/cloud-monitoring/datasource.ts
View File

@ -14,8 +14,10 @@ import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { CloudMonitoringOptions, CloudMonitoringQuery, Filter, MetricDescriptor, QueryType } from './types';
import { cloudMonitoringUnitMappings } from './constants';
import API from './api';
import API, { PostResponse } from './api';
import { CloudMonitoringVariableSupport } from './variables';
import { catchError, map, mergeMap } from 'rxjs/operators';
import { from, Observable, of, throwError } from 'rxjs';
export default class CloudMonitoringDatasource extends DataSourceApi<CloudMonitoringQuery, CloudMonitoringOptions> {
api: API;
@ -37,16 +39,23 @@ export default class CloudMonitoringDatasource extends DataSourceApi<CloudMonito
return this.templateSrv.getVariables().map(v => `$${v.name}`);
}
async query(options: DataQueryRequest<CloudMonitoringQuery>): Promise<DataQueryResponseData> {
const result: DataQueryResponseData[] = [];
const data = await this.getTimeSeries(options);
if (data.results) {
Object.values(data.results).forEach((queryRes: any) => {
if (!queryRes.series) {
return;
query(options: DataQueryRequest<CloudMonitoringQuery>): Observable<DataQueryResponseData> {
return this.getTimeSeries(options).pipe(
map(data => {
if (!data.results) {
return { data: [] };
}
const result: DataQueryResponseData[] = [];
const values = Object.values(data.results);
for (const queryRes of values) {
if (!queryRes.series) {
continue;
}
const unit = this.resolvePanelUnitFromTargets(options.targets);
queryRes.series.forEach((series: any) => {
for (const series of queryRes.series) {
let timeSerie: any = {
target: series.name,
datapoints: series.points,
@ -70,14 +79,14 @@ export default class CloudMonitoringDatasource extends DataSourceApi<CloudMonito
}
}
result.push(df);
});
});
return { data: result };
} else {
return { data: [] };
}
}
return { data: result };
})
);
}
async annotationQuery(options: any) {
await this.ensureGCEDefaultProject();
const annotation = options.annotation;
@ -101,12 +110,14 @@ export default class CloudMonitoringDatasource extends DataSourceApi<CloudMonito
},
];
const { data } = await this.api.post({
return this.api
.post({
from: options.range.from.valueOf().toString(),
to: options.range.to.valueOf().toString(),
queries,
});
})
.pipe(
map(({ data }) => {
const results = data.results['annotationQuery'].tables[0].rows.map((v: any) => {
return {
annotation: annotation,
@ -118,30 +129,38 @@ export default class CloudMonitoringDatasource extends DataSourceApi<CloudMonito
});
return results;
})
)
.toPromise();
}
async getTimeSeries(options: DataQueryRequest<CloudMonitoringQuery>) {
await this.ensureGCEDefaultProject();
getTimeSeries(options: DataQueryRequest<CloudMonitoringQuery>): Observable<PostResponse> {
const queries = options.targets
.map(this.migrateQuery)
.filter(this.shouldRunQuery)
.map(q => this.prepareTimeSeriesQuery(q, options.scopedVars))
.map(q => ({ ...q, intervalMs: options.intervalMs, type: 'timeSeriesQuery' }));
if (queries.length > 0) {
const { data } = await this.api.post({
if (!queries.length) {
return of({ results: [] });
}
return from(this.ensureGCEDefaultProject()).pipe(
mergeMap(() => {
return this.api.post({
from: options.range.from.valueOf().toString(),
to: options.range.to.valueOf().toString(),
queries,
});
}),
map(({ data }) => {
return data;
} else {
return { results: [] };
}
})
);
}
async getLabels(metricType: string, refId: string, projectName: string, groupBys?: string[]) {
const response = await this.getTimeSeries({
return this.getTimeSeries({
targets: [
{
refId,
@ -157,9 +176,14 @@ export default class CloudMonitoringDatasource extends DataSourceApi<CloudMonito
},
],
range: this.timeSrv.timeRange(),
} as DataQueryRequest<CloudMonitoringQuery>);
} as DataQueryRequest<CloudMonitoringQuery>)
.pipe(
map(response => {
const result = response.results[refId];
return result && result.meta ? result.meta.labels : {};
})
)
.toPromise();
}
async testDatasource() {
@ -205,14 +229,17 @@ export default class CloudMonitoringDatasource extends DataSourceApi<CloudMonito
},
],
})
.then(({ data }) => {
.pipe(
map(({ data }) => {
return data && data.results && data.results.getGCEDefaultProject && data.results.getGCEDefaultProject.meta
? data.results.getGCEDefaultProject.meta.defaultProject
: '';
}),
catchError(err => {
return throwError(err.data.error);
})
.catch(err => {
throw err.data.error;
});
)
.toPromise();
}
getDefaultProject(): string {
@ -272,7 +299,7 @@ export default class CloudMonitoringDatasource extends DataSourceApi<CloudMonito
});
}
async getProjects() {
getProjects() {
return this.api.get(`projects`, {
responseMap: ({ projectId, name }: { projectId: string; name: string }) => ({
value: projectId,

196
public/app/plugins/datasource/cloud-monitoring/specs/datasource.test.ts
View File

@ -1,86 +1,79 @@
import { of, throwError } from 'rxjs';
import { DataSourceInstanceSettings, observableTester, toUtc } from '@grafana/data';
import CloudMonitoringDataSource from '../datasource';
import { metricDescriptors } from './testData';
import { TemplateSrv } from 'app/features/templating/template_srv';
import { DataSourceInstanceSettings, toUtc } from '@grafana/data';
import { CloudMonitoringOptions } from '../types';
import { backendSrv } from 'app/core/services/backend_srv'; // will use the version in __mocks__
import { TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { CustomVariableModel } from '../../../../features/variables/types';
import { initialCustomVariableModelState } from '../../../../features/variables/custom/reducer';
import { createFetchResponse } from 'test/helpers/createFetchResponse';
jest.mock('@grafana/runtime', () => ({
...((jest.requireActual('@grafana/runtime') as unknown) as object),
getBackendSrv: () => backendSrv,
}));
interface Result {
status: any;
message?: any;
}
type Args = { response?: any; throws?: boolean; templateSrv?: TemplateSrv };
function getTestcontext({ response = {}, throws = false, templateSrv = new TemplateSrv() }: Args = {}) {
jest.clearAllMocks();
describe('CloudMonitoringDataSource', () => {
const instanceSettings = ({
jsonData: {
defaultProject: 'testproject',
},
} as unknown) as DataSourceInstanceSettings<CloudMonitoringOptions>;
const templateSrv = new TemplateSrv();
const timeSrv = {} as TimeSrv;
const datasourceRequestMock = jest.spyOn(backendSrv, 'datasourceRequest');
beforeEach(() => {
jest.clearAllMocks();
datasourceRequestMock.mockImplementation(jest.fn());
});
const fetchMock = jest.spyOn(backendSrv, 'fetch');
throws
? fetchMock.mockImplementation(() => throwError(response))
: fetchMock.mockImplementation(() => of(createFetchResponse(response)));
const ds = new CloudMonitoringDataSource(instanceSettings, templateSrv, timeSrv);
return { ds };
}
describe('CloudMonitoringDataSource', () => {
describe('when performing testDataSource', () => {
describe('and call to cloud monitoring api succeeds', () => {
let ds;
let result: Result;
beforeEach(async () => {
datasourceRequestMock.mockImplementation(() => Promise.resolve({ status: 200 }));
ds = new CloudMonitoringDataSource(instanceSettings, templateSrv, timeSrv);
result = await ds.testDatasource();
});
it('should return successfully', async () => {
const { ds } = getTestcontext();
const result = await ds.testDatasource();
it('should return successfully', () => {
expect(result.status).toBe('success');
});
});
describe('and a list of metricDescriptors are returned', () => {
let ds;
let result: Result;
beforeEach(async () => {
datasourceRequestMock.mockImplementation(() => Promise.resolve({ status: 200, data: metricDescriptors }));
it('should return status success', async () => {
const { ds } = getTestcontext({ response: metricDescriptors });
ds = new CloudMonitoringDataSource(instanceSettings, templateSrv, timeSrv);
result = await ds.testDatasource();
});
const result = await ds.testDatasource();
it('should return status success', () => {
expect(result.status).toBe('success');
});
});
describe('and call to cloud monitoring api fails with 400 error', () => {
let ds;
let result: Result;
beforeEach(async () => {
datasourceRequestMock.mockImplementation(() =>
Promise.reject({
it('should return error status and a detailed error message', async () => {
const response = {
statusText: 'Bad Request',
data: {
error: { code: 400, message: 'Field interval.endTime had an invalid value' },
},
})
);
};
const { ds } = getTestcontext({ response, throws: true });
ds = new CloudMonitoringDataSource(instanceSettings, templateSrv, timeSrv);
result = await ds.testDatasource();
});
const result = await ds.testDatasource();
it('should return error status and a detailed error message', () => {
expect(result.status).toEqual('error');
expect(result.message).toBe(
'Google Cloud Monitoring: Bad Request: 400. Field interval.endTime had an invalid value'
@ -90,6 +83,8 @@ describe('CloudMonitoringDataSource', () => {
});
describe('When performing query', () => {
describe('and no time series data is returned', () => {
it('should return a list of datapoints', done => {
const options = {
range: {
from: toUtc('2017-08-22T20:00:00Z'),
@ -106,8 +101,6 @@ describe('CloudMonitoringDataSource', () => {
],
};
describe('and no time series data is returned', () => {
let ds: CloudMonitoringDataSource;
const response: any = {
results: {
A: {
@ -121,27 +114,23 @@ describe('CloudMonitoringDataSource', () => {
},
};
beforeEach(() => {
datasourceRequestMock.mockImplementation(() => Promise.resolve({ status: 200, data: response }));
ds = new CloudMonitoringDataSource(instanceSettings, templateSrv, timeSrv);
});
const { ds } = getTestcontext({ response });
it('should return a list of datapoints', () => {
return ds.query(options as any).then(results => {
observableTester().subscribeAndExpectOnNext({
expect: results => {
expect(results.data.length).toBe(0);
},
observable: ds.query(options as any),
done,
});
});
});
});
describe('when performing getMetricTypes', () => {
describe('and call to cloud monitoring api succeeds', () => {});
let ds;
let result: any;
beforeEach(async () => {
datasourceRequestMock.mockImplementation(() =>
Promise.resolve({
data: {
describe('and call to cloud monitoring api succeeds', () => {
it('should return successfully', async () => {
const response = {
metricDescriptors: [
{
displayName: 'test metric name 1',
@ -152,16 +141,11 @@ describe('CloudMonitoringDataSource', () => {
type: 'logging.googleapis.com/user/logbased-metric-with-no-display-name',
},
],
},
})
);
};
const { ds } = getTestcontext({ response });
ds = new CloudMonitoringDataSource(instanceSettings, templateSrv, timeSrv);
// @ts-ignore
result = await ds.getMetricTypes('proj');
});
const result = await ds.getMetricTypes('proj');
it('should return successfully', () => {
expect(result.length).toBe(2);
expect(result[0].service).toBe('compute.googleapis.com');
expect(result[0].serviceShortName).toBe('compute');
@ -172,72 +156,60 @@ describe('CloudMonitoringDataSource', () => {
expect(result[1].displayName).toBe('logging.googleapis.com/user/logbased-metric-with-no-display-name');
});
});
describe('when interpolating a template variable for the filter', () => {
let interpolated: any[];
describe('and is single value variable', () => {
beforeEach(() => {
const filterTemplateSrv = initTemplateSrv('filtervalue1');
const ds = new CloudMonitoringDataSource(instanceSettings, filterTemplateSrv, timeSrv);
interpolated = ds.interpolateFilters(['resource.label.zone', '=~', '${test}'], {});
});
describe('when interpolating a template variable for the filter', () => {
describe('and is single value variable', () => {
it('should replace the variable with the value', () => {
const templateSrv = initTemplateSrv('filtervalue1');
const { ds } = getTestcontext({ templateSrv });
const interpolated = ds.interpolateFilters(['resource.label.zone', '=~', '${test}'], {});
expect(interpolated.length).toBe(3);
expect(interpolated[2]).toBe('filtervalue1');
});
});
describe('and is single value variable for the label part', () => {
beforeEach(() => {
const filterTemplateSrv = initTemplateSrv('resource.label.zone');
const ds = new CloudMonitoringDataSource(instanceSettings, filterTemplateSrv, timeSrv);
interpolated = ds.interpolateFilters(['${test}', '=~', 'europe-north-1a'], {});
});
it('should replace the variable with the value and not with regex formatting', () => {
const templateSrv = initTemplateSrv('resource.label.zone');
const { ds } = getTestcontext({ templateSrv });
const interpolated = ds.interpolateFilters(['${test}', '=~', 'europe-north-1a'], {});
expect(interpolated.length).toBe(3);
expect(interpolated[0]).toBe('resource.label.zone');
});
});
describe('and is multi value variable', () => {
beforeEach(() => {
const filterTemplateSrv = initTemplateSrv(['filtervalue1', 'filtervalue2'], true);
const ds = new CloudMonitoringDataSource(instanceSettings, filterTemplateSrv, timeSrv);
interpolated = ds.interpolateFilters(['resource.label.zone', '=~', '[[test]]'], {});
});
it('should replace the variable with a regex expression', () => {
const templateSrv = initTemplateSrv(['filtervalue1', 'filtervalue2'], true);
const { ds } = getTestcontext({ templateSrv });
const interpolated = ds.interpolateFilters(['resource.label.zone', '=~', '[[test]]'], {});
expect(interpolated[2]).toBe('(filtervalue1|filtervalue2)');
});
});
});
describe('when interpolating a template variable for group bys', () => {
let interpolated: any[];
describe('and is single value variable', () => {
beforeEach(() => {
const groupByTemplateSrv = initTemplateSrv('groupby1');
const ds = new CloudMonitoringDataSource(instanceSettings, groupByTemplateSrv, timeSrv);
interpolated = ds.interpolateGroupBys(['[[test]]'], {});
});
it('should replace the variable with the value', () => {
const templateSrv = initTemplateSrv('groupby1');
const { ds } = getTestcontext({ templateSrv });
const interpolated = ds.interpolateGroupBys(['[[test]]'], {});
expect(interpolated.length).toBe(1);
expect(interpolated[0]).toBe('groupby1');
});
});
describe('and is multi value variable', () => {
beforeEach(() => {
const groupByTemplateSrv = initTemplateSrv(['groupby1', 'groupby2'], true);
const ds = new CloudMonitoringDataSource(instanceSettings, groupByTemplateSrv, timeSrv);
interpolated = ds.interpolateGroupBys(['[[test]]'], {});
});
it('should replace the variable with an array of group bys', () => {
const templateSrv = initTemplateSrv(['groupby1', 'groupby2'], true);
const { ds } = getTestcontext({ templateSrv });
const interpolated = ds.interpolateGroupBys(['[[test]]'], {});
expect(interpolated.length).toBe(2);
expect(interpolated[0]).toBe('groupby1');
expect(interpolated[1]).toBe('groupby2');
@ -246,24 +218,20 @@ describe('CloudMonitoringDataSource', () => {
});
describe('unit parsing', () => {
let ds: CloudMonitoringDataSource, res: any;
beforeEach(() => {
ds = new CloudMonitoringDataSource(instanceSettings, templateSrv, timeSrv);
});
const { ds } = getTestcontext();
describe('when theres only one target', () => {
describe('and the cloud monitoring unit does nott have a corresponding grafana unit', () => {
beforeEach(() => {
res = ds.resolvePanelUnitFromTargets([{ unit: 'megaseconds' }]);
});
it('should return undefined', () => {
const res = ds.resolvePanelUnitFromTargets([{ unit: 'megaseconds' }]);
expect(res).toBeUndefined();
});
});
describe('and the cloud monitoring unit has a corresponding grafana unit', () => {
beforeEach(() => {
res = ds.resolvePanelUnitFromTargets([{ unit: 'bit' }]);
});
it('should return bits', () => {
const res = ds.resolvePanelUnitFromTargets([{ unit: 'bit' }]);
expect(res).toEqual('bits');
});
});
@ -271,32 +239,30 @@ describe('CloudMonitoringDataSource', () => {
describe('when theres more than one target', () => {
describe('and all target units are the same', () => {
beforeEach(() => {
res = ds.resolvePanelUnitFromTargets([{ unit: 'bit' }, { unit: 'bit' }]);
});
it('should return bits', () => {
const res = ds.resolvePanelUnitFromTargets([{ unit: 'bit' }, { unit: 'bit' }]);
expect(res).toEqual('bits');
});
});
describe('and all target units are the same but does not have grafana mappings', () => {
beforeEach(() => {
res = ds.resolvePanelUnitFromTargets([{ unit: 'megaseconds' }, { unit: 'megaseconds' }]);
});
it('should return the default value of undefined', () => {
const res = ds.resolvePanelUnitFromTargets([{ unit: 'megaseconds' }, { unit: 'megaseconds' }]);
expect(res).toBeUndefined();
});
});
describe('and all target units are not the same', () => {
beforeEach(() => {
res = ds.resolvePanelUnitFromTargets([{ unit: 'bit' }, { unit: 'min' }]);
});
it('should return the default value of undefined', () => {
const res = ds.resolvePanelUnitFromTargets([{ unit: 'bit' }, { unit: 'min' }]);
expect(res).toBeUndefined();
});
});
});
});
});
function initTemplateSrv(values: any, multi = false) {
const templateSrv = new TemplateSrv();
const test: CustomVariableModel = {

15
public/test/helpers/createFetchResponse.ts Normal file
View File

@ -0,0 +1,15 @@
import { FetchResponse } from '@grafana/runtime';
export function createFetchResponse<T>(data: T): FetchResponse<T> {
return {
data,
status: 200,
url: 'http://localhost:3000/api/query',
config: { url: 'http://localhost:3000/api/query' },
type: 'basic',
statusText: 'Ok',
redirected: false,
headers: ({} as unknown) as Headers,
ok: true,
};
}