From d62926b5a3068a11341f0902ea1d50824635492e Mon Sep 17 00:00:00 2001 From: Steven Vachon Date: Mon, 8 Jun 2020 09:52:34 -0400 Subject: [PATCH] @grafana/e2e: improvements (#25342) * Minor changes * Remove console.* logger plugin ... as it doesn't work in Electron * Only open/close panel editor options and groups when state is inverted ... meaning, only open when closed and only close when open. This avoids unpredictable states, causing inconsistent results. * Support for adding multiple datasources and dashboards ... and having them all auto-removed when tests are completed * Avoid page errors when removing dashboards and datasources [keep?] * Wait for chart data before saving panel ... so that everything is ready when returning to the dashboard --- .../src/selectors/pages.ts | 3 + packages/grafana-e2e/cypress/plugins/index.js | 4 -- .../grafana-e2e/cypress/support/commands.ts | 1 - packages/grafana-e2e/package.json | 1 - .../grafana-e2e/src/flows/addDashboard.ts | 47 +++++++++--- .../grafana-e2e/src/flows/addDataSource.ts | 62 ++++++++-------- packages/grafana-e2e/src/flows/addPanel.ts | 71 +++++++++++++++++-- .../grafana-e2e/src/flows/deleteDashboard.ts | 30 ++++++-- .../grafana-e2e/src/flows/deleteDataSource.ts | 30 ++++++-- packages/grafana-e2e/src/flows/index.ts | 2 - .../grafana-e2e/src/flows/saveNewDashboard.ts | 14 ---- packages/grafana-e2e/src/support/index.ts | 5 +- .../grafana-e2e/src/support/localStorage.ts | 23 ++++++ packages/grafana-e2e/src/support/scenario.ts | 11 +-- .../src/support/scenarioContext.ts | 31 ++++++-- 15 files changed, 240 insertions(+), 95 deletions(-) delete mode 100644 packages/grafana-e2e/src/flows/saveNewDashboard.ts create mode 100644 packages/grafana-e2e/src/support/localStorage.ts diff --git a/packages/grafana-e2e-selectors/src/selectors/pages.ts b/packages/grafana-e2e-selectors/src/selectors/pages.ts index d02c4c346cc..88263a25dbe 100644 --- a/packages/grafana-e2e-selectors/src/selectors/pages.ts +++ b/packages/grafana-e2e-selectors/src/selectors/pages.ts @@ -6,6 +6,9 @@ export const Pages = { submit: 'Login button', skip: 'Skip change password button', }, + Home: { + url: '/', + }, DataSource: { name: 'Data source settings page name input field', delete: 'Data source settings page Delete button', diff --git a/packages/grafana-e2e/cypress/plugins/index.js b/packages/grafana-e2e/cypress/plugins/index.js index 7c2120b99b0..c671748a639 100644 --- a/packages/grafana-e2e/cypress/plugins/index.js +++ b/packages/grafana-e2e/cypress/plugins/index.js @@ -2,21 +2,17 @@ const compareScreenshots = require('./compareScreenshots'); const extendConfig = require('./extendConfig'); const readProvisions = require('./readProvisions'); const typescriptPreprocessor = require('./typescriptPreprocessor'); -const { install: installConsoleLogger } = require('cypress-log-to-output'); module.exports = (on, config) => { on('file:preprocessor', typescriptPreprocessor); on('task', { compareScreenshots, readProvisions }); on('task', { - // @todo remove log({ message, optional }) { optional ? console.log(message, optional) : console.log(message); return null; }, }); - installConsoleLogger(on); - // Always extend with this library's config and return for diffing // @todo remove this when possible: https://github.com/cypress-io/cypress/issues/5674 return extendConfig(config); diff --git a/packages/grafana-e2e/cypress/support/commands.ts b/packages/grafana-e2e/cypress/support/commands.ts index 9c91b079f32..ae25125acf1 100644 --- a/packages/grafana-e2e/cypress/support/commands.ts +++ b/packages/grafana-e2e/cypress/support/commands.ts @@ -11,7 +11,6 @@ Cypress.Commands.add('compareScreenshots', (config: CompareScreenshotsConfig | s }); }); -// @todo remove Cypress.Commands.add('logToConsole', (message: string, optional?: any) => { cy.task('log', { message, optional }); }); diff --git a/packages/grafana-e2e/package.json b/packages/grafana-e2e/package.json index eb0c6f86dbe..fc7f55645e5 100644 --- a/packages/grafana-e2e/package.json +++ b/packages/grafana-e2e/package.json @@ -50,7 +50,6 @@ "blink-diff": "1.0.13", "commander": "5.0.0", "cypress": "^4.7.0", - "cypress-log-to-output": "^1.0.8", "execa": "4.0.0", "resolve-as-bin": "2.1.0", "ts-loader": "6.2.1", diff --git a/packages/grafana-e2e/src/flows/addDashboard.ts b/packages/grafana-e2e/src/flows/addDashboard.ts index 04da89bbf8d..1373327b4e9 100644 --- a/packages/grafana-e2e/src/flows/addDashboard.ts +++ b/packages/grafana-e2e/src/flows/addDashboard.ts @@ -1,19 +1,50 @@ +import { DeleteDashboardConfig } from './deleteDashboard'; import { e2e } from '../index'; import { getDashboardUid } from '../support/url'; -export const addDashboard = () => { - e2e().logToConsole('Adding dashboard'); +export interface AddDashboardConfig { + title: string; +} + +// @todo this actually returns type `Cypress.Chainable` +export const addDashboard = (config?: Partial): any => { + const fullConfig = { + title: `e2e-${Date.now()}`, + ...config, + } as AddDashboardConfig; + + const { title } = fullConfig; + + e2e().logToConsole('Adding dashboard with title:', title); + e2e.pages.AddDashboard.visit(); - const dashboardTitle = e2e.flows.saveNewDashboard(); - e2e().logToConsole('Added dashboard with title:', dashboardTitle); + e2e.pages.Dashboard.Toolbar.toolbarItems('Save dashboard').click(); - e2e() + e2e.pages.SaveDashboardAsModal.newName() + .clear() + .type(title); + e2e.pages.SaveDashboardAsModal.save().click(); + + e2e.flows.assertSuccessNotification(); + + e2e().logToConsole('Added dashboard with title:', title); + + return e2e() .url() .then((url: string) => { - e2e.setScenarioContext({ - lastAddedDashboard: dashboardTitle, - lastAddedDashboardUid: getDashboardUid(url), + const uid = getDashboardUid(url); + + e2e.getScenarioContext().then(({ addedDashboards }: any) => { + e2e.setScenarioContext({ + addedDashboards: [...addedDashboards, { title, uid } as DeleteDashboardConfig], + }); + }); + + // @todo remove `wrap` when possible + return e2e().wrap({ + config: fullConfig, + uid, }); }); }; diff --git a/packages/grafana-e2e/src/flows/addDataSource.ts b/packages/grafana-e2e/src/flows/addDataSource.ts index de7a176b733..5b4c299bbe6 100644 --- a/packages/grafana-e2e/src/flows/addDataSource.ts +++ b/packages/grafana-e2e/src/flows/addDataSource.ts @@ -1,64 +1,68 @@ +import { DeleteDataSourceConfig } from './deleteDataSource'; import { e2e } from '../index'; import { fromBaseUrl, getDataSourceId } from '../support/url'; -import { setScenarioContext } from '../support/scenarioContext'; export interface AddDataSourceConfig { checkHealth: boolean; expectedAlertMessage: string | RegExp; form: Function; name: string; + type: string; } -const DEFAULT_ADD_DATA_SOURCE_CONFIG: AddDataSourceConfig = { - checkHealth: false, - expectedAlertMessage: 'Data source is working', - form: () => {}, - name: 'TestData DB', -}; +// @todo this actually returns type `Cypress.Chainable` +export const addDataSource = (config?: Partial): any => { + const fullConfig = { + checkHealth: false, + expectedAlertMessage: 'Data source is working', + form: () => {}, + name: `e2e-${Date.now()}`, + type: 'TestData DB', + ...config, + } as AddDataSourceConfig; -export const addDataSource = (config?: Partial): string => { - const { checkHealth, expectedAlertMessage, form, name } = { ...DEFAULT_ADD_DATA_SOURCE_CONFIG, ...config }; + const { checkHealth, expectedAlertMessage, form, name, type } = fullConfig; e2e().logToConsole('Adding data source with name:', name); e2e.pages.AddDataSource.visit(); - e2e.pages.AddDataSource.dataSourcePlugins(name) + e2e.pages.AddDataSource.dataSourcePlugins(type) .scrollIntoView() .should('be.visible') // prevents flakiness .click(); - const dataSourceName = `e2e-${Date.now()}`; e2e.pages.DataSource.name().clear(); - e2e.pages.DataSource.name().type(dataSourceName); + e2e.pages.DataSource.name().type(name); form(); e2e.pages.DataSource.saveAndTest().click(); e2e.pages.DataSource.alert().should('exist'); e2e.pages.DataSource.alertMessage().contains(expectedAlertMessage); // assertion - e2e().logToConsole('Added data source with name:', dataSourceName); + e2e().logToConsole('Added data source with name:', name); - if (checkHealth) { - e2e() - .url() - .then((url: string) => { - const dataSourceId = getDataSourceId(url); + return e2e() + .url() + .then((url: string) => { + const id = getDataSourceId(url); - setScenarioContext({ - lastAddedDataSource: dataSourceName, - lastAddedDataSourceId: dataSourceId, + e2e.getScenarioContext().then(({ addedDataSources }: any) => { + e2e.setScenarioContext({ + addedDataSources: [...addedDataSources, { id, name } as DeleteDataSourceConfig], }); + }); - const healthUrl = fromBaseUrl(`/api/datasources/${dataSourceId}/health`); + if (checkHealth) { + const healthUrl = fromBaseUrl(`/api/datasources/${id}/health`); e2e().logToConsole(`Fetching ${healthUrl}`); e2e() .request(healthUrl) .its('body') .should('have.property', 'status') .and('eq', 'OK'); - }); - } else { - setScenarioContext({ - lastAddedDataSource: dataSourceName, - }); - } + } - return dataSourceName; + // @todo remove `wrap` when possible + return e2e().wrap({ + config: fullConfig, + id, + }); + }); }; diff --git a/packages/grafana-e2e/src/flows/addPanel.ts b/packages/grafana-e2e/src/flows/addPanel.ts index e6f6f746203..83d7bbf3b74 100644 --- a/packages/grafana-e2e/src/flows/addPanel.ts +++ b/packages/grafana-e2e/src/flows/addPanel.ts @@ -1,4 +1,5 @@ import { e2e } from '../index'; +import { getLocalStorage, requireLocalStorage } from '../support/localStorage'; import { getScenarioContext } from '../support/scenarioContext'; export interface AddPanelConfig { @@ -7,6 +8,7 @@ export interface AddPanelConfig { queriesForm: (config: AddPanelConfig) => void; panelTitle: string; visualizationName: string; + waitForChartData: boolean; } // @todo this actually returns type `Cypress.Chainable` @@ -18,6 +20,7 @@ export const addPanel = (config?: Partial): any => panelTitle: `e2e-${Date.now()}`, queriesForm: () => {}, visualizationName: 'Table', + waitForChartData: true, ...config, } as AddPanelConfig; @@ -31,42 +34,98 @@ export const addPanel = (config?: Partial): any => .get('.ds-picker') .click() .contains('[id^="react-select-"][id*="-option-"]', dataSourceName) + .scrollIntoView() .click(); + isOptionsOpen().then((isOpen: any) => { + if (!isOpen) { + toggleOptions(); + } + }); + + openOptionsGroup('settings'); getOptionsGroup('settings') .find('[value="Panel Title"]') + .scrollIntoView() .clear() .type(panelTitle); - toggleOptionsGroup('settings'); + closeOptionsGroup('settings'); - toggleOptionsGroup('type'); + openOptionsGroup('type'); e2e() .get(`[aria-label="Plugin visualization item ${visualizationName}"]`) .scrollIntoView() .click(); - toggleOptionsGroup('type'); + closeOptionsGroup('type'); + + e2e().server(); + e2e() + .route('POST', '/api/ds/query') + .as('chartData'); queriesForm(fullConfig); + e2e().wait('@chartData'); + // @todo enable when plugins have this implemented //e2e.components.QueryEditorRow.actionButton('Disable/enable query').click(); //e2e.components.Panels.Panel.containerByTitle(panelTitle).find('.panel-content').contains('No data'); //e2e.components.QueryEditorRow.actionButton('Disable/enable query').click(); - e2e.components.PanelEditor.OptionsPane.close().click(); + isOptionsOpen().then((isOpen: any) => { + if (isOpen) { + toggleOptions(); + } + }); e2e() .get('button[title="Apply changes and go back to dashboard"]') .click(); // @todo remove `wrap` when possible - return e2e().wrap(fullConfig); + return e2e().wrap({ config: fullConfig }); + }); + +// @todo this actually returns type `Cypress.Chainable` +const closeOptionsGroup = (name: string): any => + isOptionsGroupOpen(name).then((isOpen: any) => { + if (isOpen) { + toggleOptionsGroup(name); + } }); const getOptionsGroup = (name: string) => e2e().get(`.options-group:has([aria-label="Options group Panel ${name}"])`); +// @todo this actually returns type `Cypress.Chainable` +const isOptionsGroupOpen = (name: string): any => + requireLocalStorage(`grafana.dashboard.editor.ui.optionGroup[Panel ${name}]`).then(({ defaultToClosed }: any) => { + // @todo remove `wrap` when possible + return e2e().wrap(!defaultToClosed); + }); + +// @todo this actually returns type `Cypress.Chainable` +const isOptionsOpen = (): any => + getLocalStorage('grafana.dashboard.editor.ui').then((data: any) => { + if (data) { + // @todo remove `wrap` when possible + return e2e().wrap(data.isPanelOptionsVisible); + } else { + // @todo remove `wrap` when possible + return e2e().wrap(true); + } + }); + +// @todo this actually returns type `Cypress.Chainable` +const openOptionsGroup = (name: string): any => + isOptionsGroupOpen(name).then((isOpen: any) => { + if (!isOpen) { + toggleOptionsGroup(name); + } + }); + +const toggleOptions = () => e2e.components.PanelEditor.OptionsPane.close().click(); + const toggleOptionsGroup = (name: string) => getOptionsGroup(name) .find('.editor-options-group-toggle') - .scrollIntoView() .click(); diff --git a/packages/grafana-e2e/src/flows/deleteDashboard.ts b/packages/grafana-e2e/src/flows/deleteDashboard.ts index 1bf2216520a..28238a7153c 100644 --- a/packages/grafana-e2e/src/flows/deleteDashboard.ts +++ b/packages/grafana-e2e/src/flows/deleteDashboard.ts @@ -1,12 +1,20 @@ import { e2e } from '../index'; import { fromBaseUrl } from '../support/url'; -export const deleteDashboard = (dashBoardUid: string) => { - e2e().logToConsole('Deleting dashboard with uid:', dashBoardUid); - e2e().request('DELETE', fromBaseUrl(`/api/dashboards/uid/${dashBoardUid}`)); +export interface DeleteDashboardConfig { + title: string; + uid: string; +} + +export const deleteDashboard = ({ title, uid }: DeleteDashboardConfig) => { + e2e().logToConsole('Deleting dashboard with uid:', uid); + + // Avoid dashboard page errors + e2e.pages.Home.visit(); + e2e().request('DELETE', fromBaseUrl(`/api/dashboards/uid/${uid}`)); /* https://github.com/cypress-io/cypress/issues/2831 - Flows.openDashboard(dashboardName); + Flows.openDashboard(title); Pages.Dashboard.settings().click(); @@ -20,9 +28,17 @@ export const deleteDashboard = (dashBoardUid: string) => { Pages.Dashboards.dashboards().each(item => { const text = item.text(); Cypress.log({ message: [text] }); - if (text && text.indexOf(dashboardName) !== -1) { - expect(false).equals(true, `Dashboard ${dashboardName} was found although it was deleted.`); + if (text && text.indexOf(title) !== -1) { + expect(false).equals(true, `Dashboard ${title} was found although it was deleted.`); } }); - */ + */ + + e2e.getScenarioContext().then(({ addedDashboards }: any) => { + e2e.setScenarioContext({ + addedDashboards: addedDashboards.filter((dashboard: DeleteDashboardConfig) => { + return dashboard.title !== title && dashboard.uid !== uid; + }), + }); + }); }; diff --git a/packages/grafana-e2e/src/flows/deleteDataSource.ts b/packages/grafana-e2e/src/flows/deleteDataSource.ts index 70dee352261..2ca447addc5 100644 --- a/packages/grafana-e2e/src/flows/deleteDataSource.ts +++ b/packages/grafana-e2e/src/flows/deleteDataSource.ts @@ -1,13 +1,21 @@ import { e2e } from '../index'; import { fromBaseUrl } from '../support/url'; -export const deleteDataSource = (dataSourceName: string) => { - e2e().logToConsole('Deleting data source with name:', dataSourceName); - e2e().request('DELETE', fromBaseUrl(`/api/datasources/name/${dataSourceName}`)); +export interface DeleteDataSourceConfig { + id: string; + name: string; +} + +export const deleteDataSource = ({ id, name }: DeleteDataSourceConfig) => { + e2e().logToConsole('Deleting data source with name:', name); + + // Avoid datasources page errors + e2e.pages.Home.visit(); + e2e().request('DELETE', fromBaseUrl(`/api/datasources/name/${name}`)); /* https://github.com/cypress-io/cypress/issues/2831 Pages.DataSources.visit(); - Pages.DataSources.dataSources(dataSourceName).click(); + Pages.DataSources.dataSources(name).click(); Pages.DataSource.delete().click(); @@ -16,9 +24,17 @@ export const deleteDataSource = (dataSourceName: string) => { Pages.DataSources.visit(); Pages.DataSources.dataSources().each(item => { const text = item.text(); - if (text && text.indexOf(dataSourceName) !== -1) { - expect(false).equals(true, `Data source ${dataSourceName} was found although it was deleted.`); + if (text && text.indexOf(name) !== -1) { + expect(false).equals(true, `Data source ${name} was found although it was deleted.`); } }); - */ + */ + + e2e.getScenarioContext().then(({ addedDataSources }: any) => { + e2e.setScenarioContext({ + addedDataSources: addedDataSources.filter((dataSource: DeleteDataSourceConfig) => { + return dataSource.id !== id && dataSource.name !== name; + }), + }); + }); }; diff --git a/packages/grafana-e2e/src/flows/index.ts b/packages/grafana-e2e/src/flows/index.ts index 0c2e63aa14b..976fac4c326 100644 --- a/packages/grafana-e2e/src/flows/index.ts +++ b/packages/grafana-e2e/src/flows/index.ts @@ -7,7 +7,6 @@ import { deleteDataSource } from './deleteDataSource'; import { login } from './login'; import { openDashboard } from './openDashboard'; import { saveDashboard } from './saveDashboard'; -import { saveNewDashboard } from './saveNewDashboard'; import { openPanelMenuItem, PanelMenuItems } from './openPanelMenuItem'; export const Flows = { @@ -20,7 +19,6 @@ export const Flows = { login, openDashboard, saveDashboard, - saveNewDashboard, openPanelMenuItem, PanelMenuItems, }; diff --git a/packages/grafana-e2e/src/flows/saveNewDashboard.ts b/packages/grafana-e2e/src/flows/saveNewDashboard.ts deleted file mode 100644 index 34dd3d1b648..00000000000 --- a/packages/grafana-e2e/src/flows/saveNewDashboard.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { e2e } from '../index'; - -export const saveNewDashboard = () => { - e2e.pages.Dashboard.Toolbar.toolbarItems('Save dashboard').click(); - - const dashboardTitle = `e2e-${Date.now()}`; - e2e.pages.SaveDashboardAsModal.newName().clear(); - e2e.pages.SaveDashboardAsModal.newName().type(dashboardTitle); - e2e.pages.SaveDashboardAsModal.save().click(); - - e2e.flows.assertSuccessNotification(); - - return dashboardTitle; -}; diff --git a/packages/grafana-e2e/src/support/index.ts b/packages/grafana-e2e/src/support/index.ts index 2b3cee22367..1ebbc3f881e 100644 --- a/packages/grafana-e2e/src/support/index.ts +++ b/packages/grafana-e2e/src/support/index.ts @@ -1,3 +1,4 @@ -export * from './types'; -export * from './selector'; +export * from './localStorage'; export * from './scenarioContext'; +export * from './selector'; +export * from './types'; diff --git a/packages/grafana-e2e/src/support/localStorage.ts b/packages/grafana-e2e/src/support/localStorage.ts new file mode 100644 index 00000000000..eaf3b355500 --- /dev/null +++ b/packages/grafana-e2e/src/support/localStorage.ts @@ -0,0 +1,23 @@ +import { e2e } from '../index'; + +// @todo this actually returns type `Cypress.Chainable` +const get = (key: string): any => + e2e() + .wrap({ getLocalStorage: () => localStorage.getItem(key) }) + .invoke('getLocalStorage'); + +// @todo this actually returns type `Cypress.Chainable` +export const getLocalStorage = (key: string): any => + get(key).then((value: any) => { + if (value === null) { + return value; + } else { + return JSON.parse(value); + } + }); + +// @todo this actually returns type `Cypress.Chainable` +export const requireLocalStorage = (key: string): any => + get(key) // `getLocalStorage()` would turn 'null' into `null` + .should('not.equal', null) + .then((value: any) => JSON.parse(value as string)); diff --git a/packages/grafana-e2e/src/support/scenario.ts b/packages/grafana-e2e/src/support/scenario.ts index 973ec791840..735492ea4f4 100644 --- a/packages/grafana-e2e/src/support/scenario.ts +++ b/packages/grafana-e2e/src/support/scenario.ts @@ -34,14 +34,9 @@ export const e2eScenario = ({ }); afterEach(() => { - getScenarioContext().then(({ lastAddedDashboardUid, lastAddedDataSource }: any) => { - if (lastAddedDashboardUid) { - Flows.deleteDashboard(lastAddedDashboardUid); - } - - if (lastAddedDataSource) { - Flows.deleteDataSource(lastAddedDataSource); - } + getScenarioContext().then(({ addedDashboards, addedDataSources }: any) => { + addedDashboards.forEach((dashboard: any) => Flows.deleteDashboard(dashboard)); + addedDataSources.forEach((dataSource: any) => Flows.deleteDataSource(dataSource)); }); }); diff --git a/packages/grafana-e2e/src/support/scenarioContext.ts b/packages/grafana-e2e/src/support/scenarioContext.ts index 4f9571dcf4c..17f085855b8 100644 --- a/packages/grafana-e2e/src/support/scenarioContext.ts +++ b/packages/grafana-e2e/src/support/scenarioContext.ts @@ -1,20 +1,39 @@ import { e2e } from '../index'; +import { DeleteDashboardConfig } from '../flows/deleteDashboard'; +import { DeleteDataSourceConfig } from '../flows/deleteDataSource'; export interface ScenarioContext { - lastAddedDashboard: string; + addedDashboards: DeleteDashboardConfig[]; + addedDataSources: DeleteDataSourceConfig[]; + lastAddedDashboard: string; // @todo rename to `lastAddedDashboardTitle` lastAddedDashboardUid: string; - lastAddedDataSource: string; + lastAddedDataSource: string; // @todo rename to `lastAddedDataSourceName` lastAddedDataSourceId: string; [key: string]: any; } const scenarioContext: ScenarioContext = { - lastAddedDashboard: '', - lastAddedDashboardUid: '', - lastAddedDataSource: '', - lastAddedDataSourceId: '', + addedDashboards: [], + addedDataSources: [], + get lastAddedDashboard() { + return lastProperty(this.addedDashboards, 'title'); + }, + get lastAddedDashboardUid() { + return lastProperty(this.addedDashboards, 'uid'); + }, + get lastAddedDataSource() { + return lastProperty(this.addedDataSources, 'name'); + }, + get lastAddedDataSourceId() { + return lastProperty(this.addedDataSources, 'id'); + }, }; +const lastProperty = ( + items: T[], + key: K +) => items[items.length - 1]?.[key] ?? ''; + // @todo this actually returns type `Cypress.Chainable` export const getScenarioContext = (): any => e2e()