E2E: Moving files & refactoring (#22787)

* E2E refactoring

* Updated circleci

* Updated lint paths

.circleci/config.yml

@@ -517,13 +517,13 @@ jobs:
           command: ./e2e/run-suite
           no_output_timeout: 5m
       - store_artifacts:
-          path: public/e2e-tests/screenShots/theTruth
+          path: e2e/suite1/screenshots/expected
           destination: expected-screenshots
       - store_artifacts:
-          path: public/e2e-tests/screenShots/theOutput
-          destination: output-screenshots
+          path: e2e/suite1/screenshots/received
+          destination: received-screenshots
       - store_artifacts:
-          path: public/e2e-tests/videos
+          path: e2e/suite1/videos
           destination: output-videos
       - store_artifacts:
           path: e2e/tmp/data/log

.gitignore

@@ -104,6 +104,6 @@ compilation-stats.json
 /packages/grafana-e2e/cypress/screenshots
 /packages/grafana-e2e/cypress/videos
 /packages/grafana-e2e/cypress/logs
-/public/e2e-test/screenShots/theOutput
-/public/e2e-tests/screenShots/theOutput/*.png
-/public/e2e-tests/videos
+/e2e/server.log
+/e2e/suite1/screenshots/received
+/e2e/suite1/videos/*

e2e/run-suite

@@ -4,9 +4,12 @@
 
 echo -e "Waiting for grafana-server to finish starting"
 
-timeout 200 bash -c 'until nc -z $0 $1; do sleep 1; done' localhost $PORT
+timeout 60 bash -c 'until nc -z $0 $1; do sleep 1; done' localhost $PORT
 
 echo -e "Starting Cypress scenarios"
 
-env BASE_URL=http://localhost:$PORT yarn e2e-tests
+cd packages/grafana-e2e
 
+yarn start --env BASE_URL=http://localhost:$PORT,CIRCLE_SHA1=$CIRCLE_SHA1,SLOWMO=$SLOWMO \
+  --config integrationFolder=../../e2e/suite1/specs,screenshotsFolder=../../e2e/suite1/screenshots,videosFolder=../../e2e/suite1/videos,fileServerFolder=./cypress,viewportWidth=1920,viewportHeight=1080,trashAssetsBeforeRuns=false \
+  "$@"

e2e/start-and-run-suite

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+. e2e/variables
+
+# Start it in the background
+./e2e/start-server 2>&1 > e2e/server.log &
+
+./e2e/run-suite "$@"

e2e/suite1/specs/smoketests.spec.ts

@@ -40,8 +40,8 @@ e2e.scenario({
         return;
       }
 
-      const theOutputImage = `${e2e.config().screenshotsFolder}/theOutput/smoke-test-scenario.png`;
-      const theTruthImage = `${e2e.config().screenshotsFolder}/theTruth/smoke-test-scenario.png`;
+      const theOutputImage = `${e2e.config().screenshotsFolder}/received/smoke-test-scenario.png`;
+      const theTruthImage = `${e2e.config().screenshotsFolder}/expected/smoke-test-scenario.png`;
 
       e2e().wrap(
         e2e.imgSrcToBlob(url).then((blob: any) => {

package.json

@@ -154,12 +154,11 @@
     "api-tests": "jest --notify --watch --config=devenv/e2e-api-tests/jest.js",
     "build": "grunt build",
     "dev": "webpack --progress --colors --config scripts/webpack/webpack.dev.js",
-    "e2e": "cd packages/grafana-e2e && yarn start --env BASE_URL=$BASE_URL,CIRCLE_SHA1=$CIRCLE_SHA1,SLOWMO=$SLOWMO --config integrationFolder=../../public/e2e-tests/integration,screenshotsFolder=../../public/e2e-tests/screenShots,videosFolder=../../public/e2e-tests/videos,fileServerFolder=./cypress,viewportWidth=1920,viewportHeight=1080,trashAssetsBeforeRuns=false",
-    "e2e-tests": "yarn e2e",
-    "e2e-tests:debug": "SLOWMO=1 yarn e2e --headed --no-exit",
+    "e2e": "./e2e/start-and-run-suite",
+    "e2e:debug": "SLOWMO=1 && ./e2e/start-and-run-suite --headed --no-exit",
     "jest": "jest --notify --watch",
     "jest-ci": "mkdir -p reports/junit && export JEST_JUNIT_OUTPUT_DIR=reports/junit && jest --ci --reporters=default --reporters=jest-junit --maxWorkers 2",
-    "lint": "eslint public/app public/e2e-test public/test --ext=.js,.ts,.tsx",
+    "lint": "eslint public/app e2e/suite1 public/test --ext=.js,.ts,.tsx",
     "lint:fix": "yarn lint --fix",
     "packages:build": "lerna run clean && lerna run build",
     "packages:docsExtract": "rm -rf ./scripts/docs && lerna run docsExtract",

public/e2e-test/pages/panels/editPanel.ts

@@ -1,19 +0,0 @@
-import { ClickablePageObjectType, Selector, SelectPageObjectType, TestPage } from '@grafana/toolkit/src/e2e';
-
-export interface EditPanelPage {
-  queriesTab: ClickablePageObjectType;
-  saveDashboard: ClickablePageObjectType;
-  scenarioSelect: SelectPageObjectType;
-  showXAxis: ClickablePageObjectType;
-  visualizationTab: ClickablePageObjectType;
-}
-
-export const editPanelPage = new TestPage<EditPanelPage>({
-  pageObjects: {
-    queriesTab: 'Queries tab button',
-    saveDashboard: 'Save dashboard navbar button',
-    scenarioSelect: 'Scenario Select',
-    showXAxis: () => Selector.fromSelector('[aria-label="X-Axis section"] [label=Show] .gf-form-switch'),
-    visualizationTab: 'Visualization tab button',
-  },
-});

public/e2e-test/pages/panels/panel.ts

@@ -1,13 +0,0 @@
-import { ClickablePageObjectType, TestPage } from '@grafana/toolkit/src/e2e';
-
-export interface Panel {
-  panelTitle: ClickablePageObjectType;
-  share: ClickablePageObjectType;
-}
-
-export const panel = new TestPage<Panel>({
-  pageObjects: {
-    panelTitle: 'Panel Title',
-    share: 'Share panel menu item',
-  },
-});

public/e2e-test/pages/panels/sharePanelModal.ts

@@ -1,11 +0,0 @@
-import { ClickablePageObjectType, TestPage } from '@grafana/toolkit/src/e2e';
-
-export interface SharePanelModal {
-  directLinkRenderedImage: ClickablePageObjectType;
-}
-
-export const sharePanelModal = new TestPage<SharePanelModal>({
-  pageObjects: {
-    directLinkRenderedImage: 'Link to rendered image',
-  },
-});

public/e2e-test/pages/templating/variablePage.ts

@@ -1,92 +0,0 @@
-import {
-  ClickablePageObjectType,
-  InputPageObjectType,
-  PageObjectType,
-  Selector,
-  SelectPageObjectType,
-  SwitchPageObjectType,
-  TestPage,
-} from '@grafana/toolkit/src/e2e';
-
-export interface VariablePage {
-  headerLink: ClickablePageObjectType;
-  modeLabel: PageObjectType;
-  generalNameInput: InputPageObjectType;
-  generalTypeSelect: SelectPageObjectType;
-  generalLabelInput: InputPageObjectType;
-  generalHideSelect: SelectPageObjectType;
-  queryOptionsDataSourceSelect: SelectPageObjectType;
-  queryOptionsRefreshSelect: SelectPageObjectType;
-  queryOptionsRegExInput: InputPageObjectType;
-  queryOptionsSortSelect: SelectPageObjectType;
-  queryOptionsQueryInput: InputPageObjectType;
-  selectionOptionsMultiSwitch: SwitchPageObjectType;
-  selectionOptionsIncludeAllSwitch: SwitchPageObjectType;
-  selectionOptionsCustomAllInput: InputPageObjectType;
-  valueGroupsTagsEnabledSwitch: SwitchPageObjectType;
-  valueGroupsTagsTagsQueryInput: InputPageObjectType;
-  valueGroupsTagsTagsValuesQueryInput: InputPageObjectType;
-  previewOfValuesOption: PageObjectType;
-  addButton: ClickablePageObjectType;
-  updateButton: ClickablePageObjectType;
-}
-
-export const variablePage = new TestPage<VariablePage>({
-  pageObjects: {
-    headerLink: 'Variable editor Header link',
-    modeLabel: 'Variable editor Header mode New',
-    generalNameInput: 'Variable editor Form Name field',
-    generalTypeSelect: 'Variable editor Form Type select',
-    generalLabelInput: 'Variable editor Form Label field',
-    generalHideSelect: 'Variable editor Form Hide select',
-    queryOptionsDataSourceSelect: 'Variable editor Form Query DataSource select',
-    queryOptionsRefreshSelect: 'Variable editor Form Query Refresh select',
-    queryOptionsRegExInput: 'Variable editor Form Query RegEx field',
-    queryOptionsSortSelect: 'Variable editor Form Query Sort select',
-    queryOptionsQueryInput: 'Variable editor Form Default Variable Query Editor textarea',
-    selectionOptionsMultiSwitch: () => Selector.fromSwitchLabel('Variable editor Form Multi switch'),
-    selectionOptionsIncludeAllSwitch: () => Selector.fromSwitchLabel('Variable editor Form IncludeAll switch'),
-    selectionOptionsCustomAllInput: 'Variable editor Form IncludeAll field',
-    valueGroupsTagsEnabledSwitch: () => Selector.fromSwitchLabel('Variable editor Form Query UseTags switch'),
-    valueGroupsTagsTagsQueryInput: 'Variable editor Form Query TagsQuery field',
-    valueGroupsTagsTagsValuesQueryInput: 'Variable editor Form Query TagsValuesQuery field',
-    previewOfValuesOption: 'Variable editor Preview of Values option',
-    addButton: 'Variable editor Add button',
-    updateButton: 'Variable editor Update button',
-  },
-});
-
-export interface CreateQueryVariableArguments {
-  page: TestPage<VariablePage>;
-  name: string;
-  label: string;
-  datasourceName: string;
-  query: string;
-}
-
-export const createQueryVariable = async ({
-  page,
-  name,
-  label,
-  datasourceName,
-  query,
-}: CreateQueryVariableArguments) => {
-  console.log('Creating a Query Variable with required');
-  await page.pageObjects.generalNameInput.enter(name);
-  await page.pageObjects.generalLabelInput.enter(label);
-  await page.pageObjects.queryOptionsDataSourceSelect.select(`string:${datasourceName}`);
-  await page.pageObjects.queryOptionsQueryInput.exists();
-  await page.pageObjects.queryOptionsQueryInput.containsPlaceholder('metric name or tags query');
-  await page.pageObjects.queryOptionsQueryInput.enter(query);
-  await page.pageObjects.queryOptionsQueryInput.blur();
-  await page.pageObjects.previewOfValuesOption.exists();
-  await page.pageObjects.selectionOptionsMultiSwitch.toggle();
-  await page.pageObjects.selectionOptionsMultiSwitch.isSwitchedOn();
-  await page.pageObjects.selectionOptionsIncludeAllSwitch.toggle();
-  await page.pageObjects.selectionOptionsIncludeAllSwitch.isSwitchedOn();
-  await page.pageObjects.selectionOptionsCustomAllInput.exists();
-  await page.pageObjects.selectionOptionsCustomAllInput.containsText('');
-  await page.pageObjects.selectionOptionsCustomAllInput.containsPlaceholder('blank = auto');
-  await page.pageObjects.addButton.click();
-  console.log('Creating a Query Variable with required, OK!');
-};

public/e2e-test/pages/templating/variablesPage.ts

@@ -1,49 +0,0 @@
-import { ArrayPageObjectType, ClickablePageObjectType, TestPage } from '@grafana/toolkit/src/e2e';
-
-export interface VariablesPage {
-  callToActionButton: ClickablePageObjectType;
-  variableTableNameField: ArrayPageObjectType;
-  variableTableDefinitionField: ArrayPageObjectType;
-  variableTableArrowUpButton: ArrayPageObjectType;
-  variableTableArrowDownButton: ArrayPageObjectType;
-  variableTableDuplicateButton: ArrayPageObjectType;
-  variableTableRemoveButton: ArrayPageObjectType;
-  newVariableButton: ClickablePageObjectType;
-  goBackButton: ClickablePageObjectType;
-}
-
-export const variablesPage = new TestPage<VariablesPage>({
-  pageObjects: {
-    callToActionButton: 'Call to action button Add variable',
-    variableTableNameField: 'Variable editor Table Name field',
-    variableTableDefinitionField: 'Variable editor Table Definition field',
-    variableTableArrowUpButton: 'Variable editor Table ArrowUp button',
-    variableTableArrowDownButton: 'Variable editor Table ArrowDown button',
-    variableTableDuplicateButton: 'Variable editor Table Duplicate button',
-    variableTableRemoveButton: 'Variable editor Table Remove button',
-    newVariableButton: 'Variable editor New variable button',
-    goBackButton: 'Dashboard settings Go Back button',
-  },
-});
-
-export interface AssertVariableTableArguments {
-  name: string;
-  query: string;
-}
-
-export const assertVariableTable = async (page: TestPage<VariablesPage>, args: AssertVariableTableArguments[]) => {
-  console.log('Asserting variable table');
-  await page.pageObjects.variableTableNameField.waitForSelector();
-  await page.pageObjects.variableTableNameField.hasLength(args.length);
-  await page.pageObjects.variableTableDefinitionField.hasLength(args.length);
-  await page.pageObjects.variableTableArrowUpButton.hasLength(args.length);
-  await page.pageObjects.variableTableArrowDownButton.hasLength(args.length);
-  await page.pageObjects.variableTableDuplicateButton.hasLength(args.length);
-  await page.pageObjects.variableTableRemoveButton.hasLength(args.length);
-  for (let index = 0; index < args.length; index++) {
-    const { name, query } = args[index];
-    await page.pageObjects.variableTableNameField.containsTextAtPos(`$${name}`, index);
-    await page.pageObjects.variableTableDefinitionField.containsTextAtPos(query, index);
-  }
-  console.log('Asserting variable table, Ok');
-};

public/e2e-test/scenarios/smoke.test.ts

@@ -1,81 +0,0 @@
-import { Browser, Page, Target } from 'puppeteer-core';
-
-import { compareScreenShots, constants, e2eScenario, takeScreenShot } from '@grafana/toolkit/src/e2e';
-import {
-  cleanDashboard,
-  createDashboardPage,
-  dashboardsPageFactory,
-  saveDashboardModal,
-} from '@grafana/toolkit/src/e2e/pages';
-import { panel } from 'e2e-test/pages/panels/panel';
-import { editPanelPage } from 'e2e-test/pages/panels/editPanel';
-import { sharePanelModal } from 'e2e-test/pages/panels/sharePanelModal';
-
-export const addDashboardAndSetupTestDataGraph = async (page: Page) => {
-  // Create a new Dashboard
-  const dashboardTitle = `e2e - Dashboard-${new Date().getTime()}`;
-  await createDashboardPage.init(page);
-  await createDashboardPage.navigateTo();
-  await createDashboardPage.pageObjects.addQuery.click();
-
-  await editPanelPage.init(page);
-  await editPanelPage.waitForNavigation();
-  await editPanelPage.pageObjects.queriesTab.click();
-  await editPanelPage.pageObjects.scenarioSelect.select('string:csv_metric_values');
-  await editPanelPage.pageObjects.visualizationTab.click();
-  await editPanelPage.pageObjects.showXAxis.click();
-  await editPanelPage.pageObjects.saveDashboard.click();
-
-  // Confirm save modal
-  await saveDashboardModal.init(page);
-  await saveDashboardModal.expectSelector({ selector: 'save-dashboard-as-modal' });
-  await saveDashboardModal.pageObjects.name.enter(dashboardTitle);
-  await saveDashboardModal.pageObjects.save.click();
-  await saveDashboardModal.pageObjects.success.exists();
-
-  return dashboardTitle;
-};
-
-export const clickOnSharePanelImageLinkAndCompareImages = async (
-  browser: Browser,
-  page: Page,
-  dashboardTitle: string
-) => {
-  // Share the dashboard
-  const dashboardsPage = dashboardsPageFactory(dashboardTitle);
-  await dashboardsPage.init(page);
-  await dashboardsPage.navigateTo();
-  await dashboardsPage.pageObjects.dashboard.exists();
-  await dashboardsPage.pageObjects.dashboard.click();
-
-  await panel.init(page);
-  await panel.pageObjects.panelTitle.click();
-  await panel.pageObjects.share.click();
-
-  // Verify that a new tab is opened
-  const targetPromise = new Promise(resolve => browser.once('targetcreated', resolve));
-  await sharePanelModal.init(page);
-  await sharePanelModal.pageObjects.directLinkRenderedImage.click();
-  const newTarget: Target = (await targetPromise) as Target;
-  expect(newTarget.url()).toContain(`${constants.baseUrl}/render/d-solo`);
-
-  // Take snapshot of page only when running on CircleCI
-  if (process.env.CIRCLE_SHA1) {
-    const newPage = await newTarget.page();
-    const fileName = 'smoke-test-scenario';
-    await takeScreenShot(newPage, fileName);
-    await compareScreenShots(fileName);
-  }
-};
-
-e2eScenario({
-  describeName: 'Smoke tests',
-  itName: 'Login scenario, create test data source, dashboard, panel, and export scenario',
-  skipScenario: false,
-  createTestDataSource: true,
-  scenario: async (browser: Browser, page: Page) => {
-    const dashboardTitle = await addDashboardAndSetupTestDataGraph(page);
-    await clickOnSharePanelImageLinkAndCompareImages(browser, page, dashboardTitle);
-    await cleanDashboard(page, dashboardTitle);
-  },
-});

public/e2e-test/scenarios/templating/templatevariables-crud.test.ts

@@ -1,104 +0,0 @@
-import { e2eScenario, TestPage } from '@grafana/toolkit/src/e2e';
-import { Browser, Page } from 'puppeteer-core';
-import {
-  assertVariableLabelsAndComponents,
-  DashboardPage,
-  dashboardSettingsPage,
-  saveChangesDashboardModal,
-} from '@grafana/toolkit/src/e2e/pages';
-
-import { assertVariableTable, variablesPage } from '../../pages/templating/variablesPage';
-import { createQueryVariable, variablePage } from '../../pages/templating/variablePage';
-
-e2eScenario({
-  describeName: 'Template Variables tests',
-  itName: 'Template Variables QueryVariable CRUD',
-  createTestDataSource: true,
-  createTestDashboard: true,
-  skipScenario: true,
-  scenario: async (browser: Browser, page: Page, datasourceName?: string, dashboardPage?: TestPage<DashboardPage>) => {
-    await dashboardPage?.pageObjects.settings.click();
-
-    await dashboardSettingsPage.init(page);
-    await dashboardSettingsPage.pageObjects.variablesSection.click();
-
-    await variablesPage.init(page);
-    await variablesPage.pageObjects.callToActionButton.exists();
-    await variablesPage.pageObjects.callToActionButton.click();
-
-    console.log('Asserting defaults for new variable');
-    await variablePage.init(page);
-    await variablePage.pageObjects.generalNameInput.exists();
-    await variablePage.pageObjects.generalNameInput.containsText('');
-    await variablePage.pageObjects.generalNameInput.containsPlaceholder('name');
-    await variablePage.pageObjects.generalTypeSelect.exists();
-    await variablePage.pageObjects.generalTypeSelect.selectedTextIs('Query');
-    await variablePage.pageObjects.generalLabelInput.exists();
-    await variablePage.pageObjects.generalLabelInput.containsText('');
-    await variablePage.pageObjects.generalLabelInput.containsPlaceholder('optional display name');
-    await variablePage.pageObjects.generalHideSelect.exists();
-    await variablePage.pageObjects.generalHideSelect.selectedTextIs('');
-    await variablePage.pageObjects.queryOptionsDataSourceSelect.exists();
-    await variablePage.pageObjects.queryOptionsDataSourceSelect.selectedTextIs('');
-    await variablePage.pageObjects.queryOptionsRefreshSelect.exists();
-    await variablePage.pageObjects.queryOptionsRefreshSelect.selectedTextIs('Never');
-    await variablePage.pageObjects.queryOptionsRegExInput.exists();
-    await variablePage.pageObjects.queryOptionsRegExInput.containsText('');
-    await variablePage.pageObjects.queryOptionsRegExInput.containsPlaceholder('/.*-(.*)-.*/');
-    await variablePage.pageObjects.queryOptionsSortSelect.exists();
-    await variablePage.pageObjects.queryOptionsSortSelect.selectedTextIs('Disabled');
-    await variablePage.pageObjects.selectionOptionsMultiSwitch.exists();
-    await variablePage.pageObjects.selectionOptionsMultiSwitch.isSwitchedOff();
-    await variablePage.pageObjects.selectionOptionsIncludeAllSwitch.exists();
-    await variablePage.pageObjects.selectionOptionsIncludeAllSwitch.isSwitchedOff();
-    await variablePage.pageObjects.valueGroupsTagsEnabledSwitch.exists();
-    await variablePage.pageObjects.valueGroupsTagsEnabledSwitch.isSwitchedOff();
-    console.log('Asserting defaults for new variable, OK!');
-
-    await variablesPage.pageObjects.goBackButton.click();
-
-    await dashboardPage?.pageObjects.settings.click();
-
-    await dashboardSettingsPage.init(page);
-    await dashboardSettingsPage.pageObjects.variablesSection.click();
-
-    await variablesPage.pageObjects.callToActionButton.exists();
-    await variablesPage.pageObjects.callToActionButton.click();
-
-    const queryVariables = [
-      { name: 'query1', query: '*', label: 'query1-label', options: ['All', 'A', 'B', 'C'] },
-      { name: 'query2', query: '$query1.*', label: 'query2-label', options: ['All', 'AA', 'AB', 'AC'] },
-      { name: 'query3', query: '$query1.$query2.*', label: 'query2-label', options: ['All', 'AAA', 'AAB', 'AAC'] },
-    ];
-
-    for (let queryVariableIndex = 0; queryVariableIndex < queryVariables.length; queryVariableIndex++) {
-      const { name, label, query } = queryVariables[queryVariableIndex];
-      const asserts = queryVariables.slice(0, queryVariableIndex + 1);
-      await createQueryVariable({
-        page: variablePage,
-        datasourceName: datasourceName as string,
-        name,
-        label,
-        query,
-      });
-
-      await assertVariableTable(variablesPage, asserts);
-
-      await dashboardSettingsPage.pageObjects.saveDashBoard.click();
-
-      await saveChangesDashboardModal.init(page);
-      await saveChangesDashboardModal.pageObjects.save.click();
-      await saveChangesDashboardModal.pageObjects.success.exists();
-
-      await variablesPage.pageObjects.goBackButton.click();
-
-      await assertVariableLabelsAndComponents(dashboardPage as TestPage<DashboardPage>, asserts);
-
-      await dashboardPage?.pageObjects.settings.click();
-
-      await dashboardSettingsPage.pageObjects.variablesSection.click();
-
-      await variablesPage.pageObjects.newVariableButton.click();
-    }
-  },
-});