Variables: Fix for migrating legacy data source properties (#43263)

This commit is contained in:
Hugo Häggmark 2021-12-20 06:49:17 +01:00 committed by GitHub
parent e240c21a43
commit aa47cac69f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 319 additions and 133 deletions

View File

@ -1815,8 +1815,8 @@ describe('DashboardModel', () => {
});
});
it('should update variable datasource props to refs', () => {
expect(model.templating.list[0].datasource).toEqual({ type: 'prometheus', uid: 'mock-ds-2' });
it('should not update variable datasource props to refs', () => {
expect(model.templating.list[0].datasource).toEqual('prom');
});
it('should update panel datasource props to refs for named data source', () => {

View File

@ -7,24 +7,24 @@ import kbn from 'app/core/utils/kbn';
import { PanelModel } from './PanelModel';
import { DashboardModel } from './DashboardModel';
import {
AnnotationQuery,
DataLink,
DataLinkBuiltInVars,
DataQuery,
DataSourceRef,
DataTransformerConfig,
getActiveThreshold,
getDataSourceRef,
isDataSourceRef,
MappingType,
SpecialValueMatch,
PanelPlugin,
SpecialValueMatch,
standardEditorsRegistry,
standardFieldConfigEditorRegistry,
ThresholdsConfig,
urlUtil,
ValueMap,
ValueMapping,
getActiveThreshold,
DataTransformerConfig,
AnnotationQuery,
DataQuery,
getDataSourceRef,
isDataSourceRef,
} from '@grafana/data';
// Constants
import {
@ -46,11 +46,11 @@ import { getDataSourceSrv } from '@grafana/runtime';
import { labelsToFieldsTransformer } from '../../../../../packages/grafana-data/src/transformations/transformers/labelsToFields';
import { mergeTransformer } from '../../../../../packages/grafana-data/src/transformations/transformers/merge';
import {
migrateMultipleStatsMetricsQuery,
migrateMultipleStatsAnnotationQuery,
migrateCloudWatchQuery,
migrateMultipleStatsAnnotationQuery,
migrateMultipleStatsMetricsQuery,
} from 'app/plugins/datasource/cloudwatch/migrations';
import { CloudWatchMetricsQuery, CloudWatchAnnotationQuery } from 'app/plugins/datasource/cloudwatch/types';
import { CloudWatchAnnotationQuery, CloudWatchMetricsQuery } from 'app/plugins/datasource/cloudwatch/types';
standardEditorsRegistry.setInit(getStandardOptionEditors);
standardFieldConfigEditorRegistry.setInit(getStandardFieldConfigs);
@ -697,13 +697,6 @@ export class DashboardMigrator {
// Replace datasource name with reference, uid and type
if (oldVersion < 33) {
for (const variable of this.dashboard.templating.list) {
if (variable.type !== 'query') {
continue;
}
variable.datasource = migrateDatasourceNameToRef(variable.datasource);
}
panelUpgrades.push((panel) => {
panel.datasource = migrateDatasourceNameToRef(panel.datasource);

View File

@ -1,6 +1,4 @@
import { cloneDeep } from 'lodash';
import { getDataSourceSrv } from '@grafana/runtime';
import { getDataSourceRef } from '@grafana/data';
import { AdHocVariableModel } from '../types';
import { dispatch } from '../../../store/store';
@ -10,7 +8,6 @@ import { adHocVariableReducer, initialAdHocVariableModelState } from './reducer'
import { AdHocVariableEditor } from './AdHocVariableEditor';
import { setFiltersFromUrl } from './actions';
import * as urlParser from './urlParser';
import { isAdHoc, isLegacyAdHocDataSource } from '../guard';
const noop = async () => {};
@ -38,24 +35,5 @@ export const createAdHocVariableAdapter = (): VariableAdapter<AdHocVariableModel
const filters = variable?.filters ?? [];
return urlParser.toUrl(filters);
},
beforeAdding: (model) => {
if (!isAdHoc(model)) {
return model;
}
if (!isLegacyAdHocDataSource(model.datasource)) {
return model;
}
const ds = getDataSourceSrv().getInstanceSettings(model.datasource);
if (!ds) {
return model;
}
const clone = cloneDeep(model);
clone.datasource = getDataSourceRef(ds);
return { ...clone };
},
};
};

View File

@ -1,6 +1,6 @@
import { AnyAction } from 'redux';
import { getRootReducer, getTemplatingRootReducer, RootReducerType, TemplatingReducerType } from './helpers';
import { getTemplatingRootReducer, TemplatingReducerType } from './helpers';
import { variableAdapters } from '../adapters';
import { createQueryVariableAdapter } from '../query/adapter';
import { createCustomVariableAdapter } from '../custom/adapter';
@ -14,7 +14,6 @@ import {
cleanUpVariables,
fixSelectedInconsistency,
initDashboardTemplating,
initVariablesTransaction,
isVariableUrlValueDifferentFromCurrent,
processVariables,
validateVariableSelectionState,
@ -47,18 +46,11 @@ import {
changeVariableNameFailed,
changeVariableNameSucceeded,
cleanEditorState,
initialVariableEditorState,
setIdInEditor,
} from '../editor/reducer';
import {
TransactionStatus,
variablesClearTransaction,
variablesCompleteTransaction,
variablesInitTransaction,
} from './transactionReducer';
import { cleanPickerState, initialState } from '../pickers/OptionsPicker/reducer';
import { variablesClearTransaction, variablesInitTransaction } from './transactionReducer';
import { cleanPickerState } from '../pickers/OptionsPicker/reducer';
import { cleanVariables } from './variablesReducer';
import { expect } from '../../../../test/lib/common';
import { ConstantVariableModel, VariableRefresh } from '../types';
import { updateVariableOptions } from '../query/reducer';
import { setVariableQueryRunner, VariableQueryRunner } from '../query/VariableQueryRunner';
@ -577,86 +569,6 @@ describe('shared actions', () => {
});
});
describe('initVariablesTransaction', () => {
function getTestContext() {
const reportSpy = jest.spyOn(runtime, 'reportInteraction').mockReturnValue(undefined);
const constant = constantBuilder().withId('constant').withName('constant').build();
const templating: any = { list: [constant] };
const uid = 'uid';
const dashboard: any = { title: 'Some dash', uid, templating };
return { reportSpy, constant, templating, uid, dashboard };
}
describe('when called and the previous dashboard has completed', () => {
it('then correct actions are dispatched', async () => {
const { constant, uid, dashboard } = getTestContext();
const tester = await reduxTester<RootReducerType>()
.givenRootReducer(getRootReducer())
.whenAsyncActionIsDispatched(initVariablesTransaction(uid, dashboard));
tester.thenDispatchedActionsPredicateShouldEqual((dispatchedActions) => {
expect(dispatchedActions[0]).toEqual(variablesInitTransaction({ uid }));
expect(dispatchedActions[1].type).toEqual(addVariable.type);
expect(dispatchedActions[1].payload.id).toEqual('__dashboard');
expect(dispatchedActions[2].type).toEqual(addVariable.type);
expect(dispatchedActions[2].payload.id).toEqual('__org');
expect(dispatchedActions[3].type).toEqual(addVariable.type);
expect(dispatchedActions[3].payload.id).toEqual('__user');
expect(dispatchedActions[4]).toEqual(
addVariable(toVariablePayload(constant, { global: false, index: 0, model: constant }))
);
expect(dispatchedActions[5]).toEqual(variableStateNotStarted(toVariablePayload(constant)));
expect(dispatchedActions[6]).toEqual(variableStateCompleted(toVariablePayload(constant)));
expect(dispatchedActions[7]).toEqual(variablesCompleteTransaction({ uid }));
return dispatchedActions.length === 8;
});
});
});
describe('when called and the previous dashboard is still processing variables', () => {
it('then correct actions are dispatched', async () => {
const { constant, uid, dashboard } = getTestContext();
const transactionState = { uid: 'previous-uid', status: TransactionStatus.Fetching };
const tester = await reduxTester<RootReducerType>({
preloadedState: ({
templating: {
transaction: transactionState,
variables: {},
optionsPicker: { ...initialState },
editor: { ...initialVariableEditorState },
},
} as unknown) as RootReducerType,
})
.givenRootReducer(getRootReducer())
.whenAsyncActionIsDispatched(initVariablesTransaction(uid, dashboard));
tester.thenDispatchedActionsPredicateShouldEqual((dispatchedActions) => {
expect(dispatchedActions[0]).toEqual(cleanVariables());
expect(dispatchedActions[1]).toEqual(cleanEditorState());
expect(dispatchedActions[2]).toEqual(cleanPickerState());
expect(dispatchedActions[3]).toEqual(variablesClearTransaction());
expect(dispatchedActions[4]).toEqual(variablesInitTransaction({ uid }));
expect(dispatchedActions[5].type).toEqual(addVariable.type);
expect(dispatchedActions[5].payload.id).toEqual('__dashboard');
expect(dispatchedActions[6].type).toEqual(addVariable.type);
expect(dispatchedActions[6].payload.id).toEqual('__org');
expect(dispatchedActions[7].type).toEqual(addVariable.type);
expect(dispatchedActions[7].payload.id).toEqual('__user');
expect(dispatchedActions[8]).toEqual(
addVariable(toVariablePayload(constant, { global: false, index: 0, model: constant }))
);
expect(dispatchedActions[9]).toEqual(variableStateNotStarted(toVariablePayload(constant)));
expect(dispatchedActions[10]).toEqual(variableStateCompleted(toVariablePayload(constant)));
expect(dispatchedActions[11]).toEqual(variablesCompleteTransaction({ uid }));
return dispatchedActions.length === 12;
});
});
});
});
describe('cleanUpVariables', () => {
describe('when called', () => {
it('then correct actions are dispatched', async () => {

View File

@ -1,6 +1,14 @@
import angular from 'angular';
import { castArray, isEqual } from 'lodash';
import { DataQuery, LoadingState, TimeRange, UrlQueryMap, UrlQueryValue } from '@grafana/data';
import {
DataQuery,
getDataSourceRef,
isDataSourceRef,
LoadingState,
TimeRange,
UrlQueryMap,
UrlQueryValue,
} from '@grafana/data';
import {
DashboardVariableModel,
@ -678,6 +686,8 @@ export const initVariablesTransaction = (dashboardUid: string, dashboard: Dashbo
dispatch(addSystemTemplateVariables(dashboard));
// Load all variables into redux store
dispatch(initDashboardTemplating(dashboard.templating.list));
// Migrate data source name to ref
dispatch(migrateVariablesDatasourceNameToRef());
// Process all variable updates
await dispatch(processVariables());
// Mark update as complete
@ -688,6 +698,31 @@ export const initVariablesTransaction = (dashboardUid: string, dashboard: Dashbo
}
};
export function migrateVariablesDatasourceNameToRef(
getDatasourceSrvFunc: typeof getDatasourceSrv = getDatasourceSrv
): ThunkResult<void> {
return function (dispatch, getState) {
const variables = getVariables(getState());
for (const variable of variables) {
if (!isAdHoc(variable) && !isQuery(variable)) {
continue;
}
const { datasource: nameOrRef } = variable;
if (isDataSourceRef(nameOrRef)) {
continue;
}
// the call to getInstanceSettings needs to be done after initDashboardTemplating because we might have
// datasource variables that need to be resolved
const ds = getDatasourceSrvFunc().getInstanceSettings(nameOrRef);
const dsRef = !ds ? { uid: nameOrRef } : getDataSourceRef(ds);
dispatch(changeVariableProp(toVariablePayload(variable, { propName: 'datasource', propValue: dsRef })));
}
};
}
export const cleanUpVariables = (): ThunkResult<void> => (dispatch) => {
dispatch(cleanVariables());
dispatch(cleanEditorState());

View File

@ -0,0 +1,190 @@
import { getRootReducer, RootReducerType } from './helpers';
import { variableAdapters } from '../adapters';
import { createQueryVariableAdapter } from '../query/adapter';
import { createConstantVariableAdapter } from '../constant/adapter';
import { reduxTester } from '../../../../test/core/redux/reduxTester';
import {
addVariable,
changeVariableProp,
setCurrentVariableValue,
variableStateCompleted,
variableStateFetching,
variableStateNotStarted,
} from './sharedReducer';
import { toVariablePayload } from './types';
import { adHocBuilder, constantBuilder, datasourceBuilder, queryBuilder } from '../shared/testing/builders';
import { cleanEditorState, initialVariableEditorState } from '../editor/reducer';
import {
TransactionStatus,
variablesClearTransaction,
variablesCompleteTransaction,
variablesInitTransaction,
} from './transactionReducer';
import { cleanPickerState, initialState } from '../pickers/OptionsPicker/reducer';
import { cleanVariables } from './variablesReducer';
import { createAdHocVariableAdapter } from '../adhoc/adapter';
import { createDataSourceVariableAdapter } from '../datasource/adapter';
import { DataSourceRef, LoadingState } from '@grafana/data/src';
import { setDataSourceSrv } from '@grafana/runtime/src';
import { VariableModel } from '../types';
import { toAsyncOfResult } from '../../query/state/DashboardQueryRunner/testHelpers';
import { setVariableQueryRunner } from '../query/VariableQueryRunner';
import { createDataSourceOptions } from '../datasource/reducer';
import { initVariablesTransaction } from './actions';
variableAdapters.setInit(() => [
createQueryVariableAdapter(),
createConstantVariableAdapter(),
createAdHocVariableAdapter(),
createDataSourceVariableAdapter(),
]);
function getTestContext(variables?: VariableModel[]) {
const uid = 'uid';
const constant = constantBuilder().withId('constant').withName('constant').build();
const templating = { list: variables ?? [constant] };
const getInstanceSettingsMock = jest.fn().mockReturnValue(undefined);
setDataSourceSrv({
get: jest.fn().mockResolvedValue({}),
getList: jest.fn().mockReturnValue([]),
getInstanceSettings: getInstanceSettingsMock,
});
const variableQueryRunner: any = {
cancelRequest: jest.fn(),
queueRequest: jest.fn(),
getResponse: () => toAsyncOfResult({ state: LoadingState.Done, identifier: { type: 'query', id: 'query' } }),
destroy: jest.fn(),
};
setVariableQueryRunner(variableQueryRunner);
const dashboard: any = { title: 'Some dash', uid, templating };
return { constant, getInstanceSettingsMock, templating, uid, dashboard };
}
describe('initVariablesTransaction', () => {
describe('when called and the previous dashboard has completed', () => {
it('then correct actions are dispatched', async () => {
const { constant, uid, dashboard } = getTestContext();
const tester = await reduxTester<RootReducerType>()
.givenRootReducer(getRootReducer())
.whenAsyncActionIsDispatched(initVariablesTransaction(uid, dashboard));
tester.thenDispatchedActionsPredicateShouldEqual((dispatchedActions) => {
expect(dispatchedActions[0]).toEqual(variablesInitTransaction({ uid }));
expect(dispatchedActions[1].type).toEqual(addVariable.type);
expect(dispatchedActions[1].payload.id).toEqual('__dashboard');
expect(dispatchedActions[2].type).toEqual(addVariable.type);
expect(dispatchedActions[2].payload.id).toEqual('__org');
expect(dispatchedActions[3].type).toEqual(addVariable.type);
expect(dispatchedActions[3].payload.id).toEqual('__user');
expect(dispatchedActions[4]).toEqual(
addVariable(toVariablePayload(constant, { global: false, index: 0, model: constant }))
);
expect(dispatchedActions[5]).toEqual(variableStateNotStarted(toVariablePayload(constant)));
expect(dispatchedActions[6]).toEqual(variableStateCompleted(toVariablePayload(constant)));
expect(dispatchedActions[7]).toEqual(variablesCompleteTransaction({ uid }));
return dispatchedActions.length === 8;
});
});
describe('and there are variables that have data source that need to be migrated', () => {
it('then correct actions are dispatched', async () => {
const legacyDs = ('${ds}' as unknown) as DataSourceRef;
const ds = datasourceBuilder().withId('ds').withName('ds').withQuery('prom').build();
const query = queryBuilder().withId('query').withName('query').withDatasource(legacyDs).build();
const adhoc = adHocBuilder().withId('adhoc').withName('adhoc').withDatasource(legacyDs).build();
const { uid, dashboard } = getTestContext([ds, query, adhoc]);
const tester = await reduxTester<RootReducerType>()
.givenRootReducer(getRootReducer())
.whenAsyncActionIsDispatched(initVariablesTransaction(uid, dashboard));
tester.thenDispatchedActionsPredicateShouldEqual((dispatchedActions) => {
expect(dispatchedActions[0]).toEqual(variablesInitTransaction({ uid }));
expect(dispatchedActions[1].type).toEqual(addVariable.type);
expect(dispatchedActions[1].payload.id).toEqual('__dashboard');
expect(dispatchedActions[2].type).toEqual(addVariable.type);
expect(dispatchedActions[2].payload.id).toEqual('__org');
expect(dispatchedActions[3].type).toEqual(addVariable.type);
expect(dispatchedActions[3].payload.id).toEqual('__user');
expect(dispatchedActions[4]).toEqual(
addVariable(toVariablePayload(ds, { global: false, index: 0, model: ds }))
);
expect(dispatchedActions[5]).toEqual(
addVariable(toVariablePayload(query, { global: false, index: 1, model: query }))
);
expect(dispatchedActions[6]).toEqual(
addVariable(toVariablePayload(adhoc, { global: false, index: 2, model: adhoc }))
);
expect(dispatchedActions[7]).toEqual(variableStateNotStarted(toVariablePayload(ds)));
expect(dispatchedActions[8]).toEqual(variableStateNotStarted(toVariablePayload(query)));
expect(dispatchedActions[9]).toEqual(variableStateNotStarted(toVariablePayload(adhoc)));
expect(dispatchedActions[10]).toEqual(
changeVariableProp(toVariablePayload(query, { propName: 'datasource', propValue: { uid: '${ds}' } }))
);
expect(dispatchedActions[11]).toEqual(
changeVariableProp(toVariablePayload(adhoc, { propName: 'datasource', propValue: { uid: '${ds}' } }))
);
expect(dispatchedActions[12]).toEqual(variableStateFetching(toVariablePayload(ds)));
expect(dispatchedActions[13]).toEqual(variableStateCompleted(toVariablePayload(adhoc)));
expect(dispatchedActions[14]).toEqual(
createDataSourceOptions(toVariablePayload(ds, { sources: [], regex: undefined }))
);
expect(dispatchedActions[15]).toEqual(
setCurrentVariableValue(
toVariablePayload(ds, { option: { selected: false, text: 'No data sources found', value: '' } })
)
);
expect(dispatchedActions[16]).toEqual(variableStateCompleted(toVariablePayload(ds)));
expect(dispatchedActions[17]).toEqual(variableStateFetching(toVariablePayload(query)));
expect(dispatchedActions[18]).toEqual(variableStateCompleted(toVariablePayload(query)));
expect(dispatchedActions[19]).toEqual(variablesCompleteTransaction({ uid }));
return dispatchedActions.length === 20;
});
});
});
});
describe('when called and the previous dashboard is still processing variables', () => {
it('then correct actions are dispatched', async () => {
const { constant, uid, dashboard } = getTestContext();
const transactionState = { uid: 'previous-uid', status: TransactionStatus.Fetching };
const tester = await reduxTester<RootReducerType>({
preloadedState: ({
templating: {
transaction: transactionState,
variables: {},
optionsPicker: { ...initialState },
editor: { ...initialVariableEditorState },
},
} as unknown) as RootReducerType,
})
.givenRootReducer(getRootReducer())
.whenAsyncActionIsDispatched(initVariablesTransaction(uid, dashboard));
tester.thenDispatchedActionsPredicateShouldEqual((dispatchedActions) => {
expect(dispatchedActions[0]).toEqual(cleanVariables());
expect(dispatchedActions[1]).toEqual(cleanEditorState());
expect(dispatchedActions[2]).toEqual(cleanPickerState());
expect(dispatchedActions[3]).toEqual(variablesClearTransaction());
expect(dispatchedActions[4]).toEqual(variablesInitTransaction({ uid }));
expect(dispatchedActions[5].type).toEqual(addVariable.type);
expect(dispatchedActions[5].payload.id).toEqual('__dashboard');
expect(dispatchedActions[6].type).toEqual(addVariable.type);
expect(dispatchedActions[6].payload.id).toEqual('__org');
expect(dispatchedActions[7].type).toEqual(addVariable.type);
expect(dispatchedActions[7].payload.id).toEqual('__user');
expect(dispatchedActions[8]).toEqual(
addVariable(toVariablePayload(constant, { global: false, index: 0, model: constant }))
);
expect(dispatchedActions[9]).toEqual(variableStateNotStarted(toVariablePayload(constant)));
expect(dispatchedActions[10]).toEqual(variableStateCompleted(toVariablePayload(constant)));
expect(dispatchedActions[11]).toEqual(variablesCompleteTransaction({ uid }));
return dispatchedActions.length === 12;
});
});
});
});

View File

@ -0,0 +1,78 @@
import { migrateVariablesDatasourceNameToRef } from './actions';
import { adHocBuilder, queryBuilder } from '../shared/testing/builders';
import { DataSourceRef } from '@grafana/data/src';
import { changeVariableProp } from './sharedReducer';
import { toVariablePayload } from './types';
function getTestContext(ds: DataSourceRef, dsInstance?: { uid: string; type: string }) {
jest.clearAllMocks();
const query = queryBuilder().withId('query').withName('query').withDatasource(ds).build();
const adhoc = adHocBuilder().withId('adhoc').withName('adhoc').withDatasource(ds).build();
const state = { templating: { variables: [query, adhoc] } };
const dispatch = jest.fn();
const getState = jest.fn().mockReturnValue(state);
const getInstanceSettingsMock = jest.fn().mockReturnValue(dsInstance);
const getDatasourceSrvFunc = jest.fn().mockReturnValue({
get: jest.fn().mockResolvedValue({}),
getList: jest.fn().mockReturnValue([]),
getInstanceSettings: getInstanceSettingsMock,
});
return { query, adhoc, dispatch, getState, getDatasourceSrvFunc };
}
describe('migrateVariablesDatasourceNameToRef', () => {
describe('when called and variables have legacy data source props', () => {
describe('and data source exists', () => {
it('then correct actions are dispatched', async () => {
const legacyDs = ('${ds}' as unknown) as DataSourceRef;
const { query, adhoc, dispatch, getState, getDatasourceSrvFunc } = getTestContext(legacyDs, {
uid: 'a random uid',
type: 'prometheus',
});
migrateVariablesDatasourceNameToRef(getDatasourceSrvFunc)(dispatch, getState, undefined);
expect(dispatch).toHaveBeenCalledTimes(2);
expect(dispatch.mock.calls[0][0]).toEqual(
changeVariableProp(
toVariablePayload(query, { propName: 'datasource', propValue: { uid: 'a random uid', type: 'prometheus' } })
)
);
expect(dispatch.mock.calls[1][0]).toEqual(
changeVariableProp(
toVariablePayload(adhoc, { propName: 'datasource', propValue: { uid: 'a random uid', type: 'prometheus' } })
)
);
});
});
describe('and data source does not exist', () => {
it('then correct actions are dispatched', async () => {
const legacyDs = ('${ds}' as unknown) as DataSourceRef;
const { query, adhoc, dispatch, getState, getDatasourceSrvFunc } = getTestContext(legacyDs, undefined);
migrateVariablesDatasourceNameToRef(getDatasourceSrvFunc)(dispatch, getState, undefined);
expect(dispatch).toHaveBeenCalledTimes(2);
expect(dispatch.mock.calls[0][0]).toEqual(
changeVariableProp(toVariablePayload(query, { propName: 'datasource', propValue: { uid: '${ds}' } }))
);
expect(dispatch.mock.calls[1][0]).toEqual(
changeVariableProp(toVariablePayload(adhoc, { propName: 'datasource', propValue: { uid: '${ds}' } }))
);
});
});
});
describe('when called and variables have dataSourceRef', () => {
it('then no actions are dispatched', async () => {
const legacyDs = { uid: '${ds}', type: 'prometheus' };
const { dispatch, getState, getDatasourceSrvFunc } = getTestContext(legacyDs, undefined);
migrateVariablesDatasourceNameToRef(getDatasourceSrvFunc)(dispatch, getState, undefined);
expect(dispatch).toHaveBeenCalledTimes(0);
});
});
});