Refactor: move end-to-end test infrastructure to @grafana/toolkit (#18012)

packages/grafana-toolkit/package.json

@@ -19,14 +19,17 @@
   },
   "author": "Grafana Labs",
   "license": "Apache-2.0",
+  "main": "src/index.ts",
   "dependencies": {
     "@babel/core": "7.4.5",
     "@babel/preset-env": "7.4.5",
     "@types/execa": "^0.9.0",
+    "@types/expect-puppeteer": "3.3.1",
     "@types/inquirer": "^6.0.3",
     "@types/jest": "24.0.13",
     "@types/jest-cli": "^23.6.0",
     "@types/node": "^12.0.4",
+    "@types/puppeteer-core": "1.9.0",
     "@types/react-dev-utils": "^9.0.1",
     "@types/semver": "^6.0.0",
     "@types/tmp": "^0.1.0",
@@ -40,6 +43,7 @@
     "copy-webpack-plugin": "5.0.3",
     "css-loader": "^3.0.0",
     "execa": "^1.0.0",
+    "expect-puppeteer": "4.1.1",
     "file-loader": "^4.0.0",
     "glob": "^7.1.4",
     "html-loader": "0.5.5",
@@ -57,6 +61,7 @@
     "postcss-loader": "3.0.0",
     "postcss-preset-env": "6.6.0",
     "prettier": "^1.18.2",
+    "puppeteer-core": "1.18.1",
     "react-dev-utils": "^9.0.1",
     "replace-in-file": "^4.1.0",
     "replace-in-file-webpack-plugin": "^1.0.6",
@@ -78,5 +83,8 @@
   "devDependencies": {
     "@types/glob": "^7.1.1",
     "@types/prettier": "^1.16.4"
+  },
+  "_moduleAliases": {
+    "puppeteer": "node_modules/puppeteer-core"
   }
 }

packages/grafana-toolkit/src/e2e/constants.ts

@@ -0,0 +1,6 @@
+export const constants = {
+  baseUrl: process.env.BASE_URL || 'http://localhost:3000',
+  chromiumRevision: '650629',
+  screenShotsTruthDir: './public/e2e-test/screenShots/theTruth',
+  screenShotsOutputDir: './public/e2e-test/screenShots/theOutput',
+};

packages/grafana-toolkit/src/e2e/images.ts

@@ -0,0 +1,84 @@
+import fs from 'fs';
+import { PNG } from 'pngjs';
+import { Page } from 'puppeteer-core';
+import pixelmatch from 'pixelmatch';
+
+import { constants } from './constants';
+
+export const takeScreenShot = async (page: Page, fileName: string) => {
+  const outputFolderExists = fs.existsSync(constants.screenShotsOutputDir);
+  if (!outputFolderExists) {
+    fs.mkdirSync(constants.screenShotsOutputDir);
+  }
+  const path = `${constants.screenShotsOutputDir}/${fileName}.png`;
+  await page.screenshot({ path, type: 'png', fullPage: false });
+};
+
+export const compareScreenShots = async (fileName: string) =>
+  new Promise(resolve => {
+    let filesRead = 0;
+
+    const doneReading = () => {
+      if (++filesRead < 2) {
+        return;
+      }
+
+      if (screenShotFromTest.width !== screenShotFromTruth.width) {
+        throw new Error(
+          `The screenshot:[${fileName}] taken during the test has a ` +
+            `width:[${screenShotFromTest.width}] that differs from the ` +
+            `expected: [${screenShotFromTruth.width}].`
+        );
+      }
+
+      if (screenShotFromTest.height !== screenShotFromTruth.height) {
+        throw new Error(
+          `The screenshot:[${fileName}] taken during the test has a ` +
+            `height:[${screenShotFromTest.height}] that differs from the ` +
+            `expected: [${screenShotFromTruth.height}].`
+        );
+      }
+
+      const diff = new PNG({ width: screenShotFromTest.width, height: screenShotFromTruth.height });
+      const numDiffPixels = pixelmatch(
+        screenShotFromTest.data,
+        screenShotFromTruth.data,
+        diff.data,
+        screenShotFromTest.width,
+        screenShotFromTest.height,
+        { threshold: 0.1 }
+      );
+
+      if (numDiffPixels !== 0) {
+        const localMessage =
+          `\nCompare the output from expected:[${constants.screenShotsTruthDir}] ` +
+          `with outcome:[${constants.screenShotsOutputDir}]`;
+        const circleCIMessage = '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.';
+        const checkMessage = process.env.CIRCLE_SHA1 ? circleCIMessage : localMessage;
+        let msg =
+          `\nThe screenshot:[${constants.screenShotsOutputDir}/${fileName}.png] ` +
+          `taken during the test differs by:[${numDiffPixels}] pixels from the expected.`;
+        msg += '\n';
+        msg += checkMessage;
+        msg += '\n';
+        msg += '\n  If the difference between expected and outcome is NOT acceptable then do the following:';
+        msg += '\n    - Check the code for changes that causes this difference, fix that and retry.';
+        msg += '\n';
+        msg += '\n  If the difference between expected and outcome is acceptable then do the following:';
+        msg += '\n    - Replace the expected image with the outcome and retry.';
+        msg += '\n';
+        throw new Error(msg);
+      }
+
+      resolve();
+    };
+
+    const screenShotFromTest = fs
+      .createReadStream(`${constants.screenShotsOutputDir}/${fileName}.png`)
+      .pipe(new PNG())
+      .on('parsed', doneReading);
+    const screenShotFromTruth = fs
+      .createReadStream(`${constants.screenShotsTruthDir}/${fileName}.png`)
+      .pipe(new PNG())
+      .on('parsed', doneReading);
+  });

packages/grafana-toolkit/src/e2e/index.ts

@@ -0,0 +1,8 @@
+export * from './constants';
+export * from './images';
+export * from './install';
+export * from './launcher';
+export * from './login';
+export * from './pageObjects';
+export * from './pages';
+export * from './scenario';

packages/grafana-toolkit/src/e2e/install.ts

@@ -0,0 +1,24 @@
+import puppeteer from 'puppeteer-core';
+import { constants } from './constants';
+
+export const downloadBrowserIfNeeded = async (): Promise<void> => {
+  const browserFetcher = puppeteer.createBrowserFetcher();
+  const localRevisions = await browserFetcher.localRevisions();
+  if (localRevisions && localRevisions.length > 0) {
+    console.log('Found a local revision for browser, exiting install.');
+    return;
+  }
+
+  console.log('Did not find any local revisions for browser, downloading latest this might take a while.');
+  await browserFetcher.download(constants.chromiumRevision, (downloaded, total) => {
+    if (downloaded === total) {
+      console.log('Chromium successfully downloaded');
+    }
+  });
+};
+
+beforeAll(async () => {
+  console.log('Checking Chromium');
+  jest.setTimeout(60 * 1000);
+  await downloadBrowserIfNeeded();
+});

packages/grafana-toolkit/src/e2e/launcher.ts

@@ -0,0 +1,29 @@
+import puppeteer, { Browser } from 'puppeteer-core';
+
+export const launchBrowser = async (): Promise<Browser> => {
+  const browserFetcher = puppeteer.createBrowserFetcher();
+  const localRevisions = await browserFetcher.localRevisions();
+  if (localRevisions.length === 0) {
+    throw new Error('Could not launch browser because there is no local revisions.');
+  }
+
+  let executablePath = null;
+  executablePath = browserFetcher.revisionInfo(localRevisions[0]).executablePath;
+
+  const browser = await puppeteer.launch({
+    headless: process.env.BROWSER ? false : true,
+    slowMo: process.env.SLOWMO ? 100 : 0,
+    defaultViewport: {
+      width: 1920,
+      height: 1080,
+      deviceScaleFactor: 1,
+      isMobile: false,
+      hasTouch: false,
+      isLandscape: false,
+    },
+    args: ['--start-fullscreen'],
+    executablePath,
+  });
+
+  return browser;
+};

packages/grafana-toolkit/src/e2e/login.ts

@@ -0,0 +1,22 @@
+import { Page } from 'puppeteer-core';
+
+import { constants } from './constants';
+import { loginPage } from './start/loginPage';
+
+export const login = async (page: Page) => {
+  await loginPage.init(page);
+  await loginPage.navigateTo();
+
+  await loginPage.pageObjects!.username.enter('admin');
+  await loginPage.pageObjects!.password.enter('admin');
+  await loginPage.pageObjects!.submit.click();
+  await loginPage.waitForResponse();
+};
+
+export const ensureLoggedIn = async (page: Page) => {
+  await page.goto(`${constants.baseUrl}`);
+  if (page.url().indexOf('login') > -1) {
+    console.log('Redirected to login page. Logging in...');
+    await login(page);
+  }
+};

packages/grafana-toolkit/src/e2e/pageObjects.ts

@@ -0,0 +1,87 @@
+import { Page } from 'puppeteer-core';
+
+export class Selector {
+  static fromAriaLabel = (selector: string) => {
+    return `[aria-label="${selector}"]`;
+  };
+
+  static fromSelector = (selector: string) => {
+    return selector;
+  };
+}
+
+export interface PageObjectType {
+  init: (page: Page) => Promise<void>;
+  exists: () => Promise<void>;
+  containsText: (text: string) => Promise<void>;
+}
+
+export interface ClickablePageObjectType extends PageObjectType {
+  click: () => Promise<void>;
+}
+
+export interface InputPageObjectType extends PageObjectType {
+  enter: (text: string) => Promise<void>;
+}
+
+export interface SelectPageObjectType extends PageObjectType {
+  select: (text: string) => Promise<void>;
+}
+
+export class PageObject implements PageObjectType {
+  protected page?: Page;
+
+  constructor(protected selector: string) {}
+
+  init = async (page: Page): Promise<void> => {
+    this.page = page;
+  };
+
+  exists = async (): Promise<void> => {
+    const options = { visible: true } as any;
+    await expect(this.page).not.toBeNull();
+    await expect(this.page).toMatchElement(this.selector, options);
+  };
+
+  containsText = async (text: string): Promise<void> => {
+    const options = { visible: true, text } as any;
+    await expect(this.page).not.toBeNull();
+    await expect(this.page).toMatchElement(this.selector, options);
+  };
+}
+
+export class ClickablePageObject extends PageObject implements ClickablePageObjectType {
+  constructor(selector: string) {
+    super(selector);
+  }
+
+  click = async (): Promise<void> => {
+    console.log('Trying to click on:', this.selector);
+    await expect(this.page).not.toBeNull();
+    await expect(this.page).toClick(this.selector);
+  };
+}
+
+export class InputPageObject extends PageObject implements InputPageObjectType {
+  constructor(selector: string) {
+    super(selector);
+  }
+
+  enter = async (text: string): Promise<void> => {
+    console.log(`Trying to enter text:${text} into:`, this.selector);
+    await expect(this.page).not.toBeNull();
+    await expect(this.page).toFill(this.selector, text);
+  };
+}
+
+export class SelectPageObject extends PageObject implements SelectPageObjectType {
+  constructor(selector: string) {
+    super(selector);
+  }
+
+  select = async (text: string): Promise<void> => {
+    console.log(`Trying to select text:${text} in dropdown:`, this.selector);
+    await expect(this.page).not.toBeNull();
+    await this.page!.select(this.selector, text);
+  };
+}

packages/grafana-toolkit/src/e2e/pages.ts

@@ -0,0 +1,111 @@
+import { Page } from 'puppeteer-core';
+import { constants } from './constants';
+import { PageObject } from './pageObjects';
+
+export interface ExpectSelectorConfig {
+  selector: string;
+  containsText?: string;
+  isVisible?: boolean;
+}
+
+export interface TestPageType<T> {
+  init: (page: Page) => Promise<void>;
+  getUrl: () => Promise<string>;
+  getUrlWithoutBaseUrl: () => Promise<string>;
+  navigateTo: () => Promise<void>;
+  expectSelector: (config: ExpectSelectorConfig) => Promise<void>;
+  waitForResponse: () => Promise<void>;
+  waitForNavigation: () => Promise<void>;
+  waitFor: (milliseconds: number) => Promise<void>;
+
+  pageObjects?: PageObjects<T>;
+}
+
+type PageObjects<T> = { [P in keyof T]: T[P] };
+
+export interface TestPageConfig<T> {
+  url?: string;
+  pageObjects?: PageObjects<T>;
+}
+
+export class TestPage<T> implements TestPageType<T> {
+  pageObjects?: PageObjects<T>;
+  private page?: Page;
+  private pageUrl?: string;
+
+  constructor(config: TestPageConfig<T>) {
+    if (config.url) {
+      this.pageUrl = `${constants.baseUrl}${config.url}`;
+    }
+    this.pageObjects = config.pageObjects;
+  }
+
+  init = async (page: Page): Promise<void> => {
+    this.page = page;
+
+    if (!this.pageObjects) {
+      return;
+    }
+
+    Object.keys(this.pageObjects).forEach(key => {
+      // @ts-ignore
+      const pageObject: PageObject = this.pageObjects[key];
+      pageObject.init(page);
+    });
+  };
+
+  navigateTo = async (): Promise<void> => {
+    this.throwIfNotInitialized();
+
+    console.log('Trying to navigate to:', this.pageUrl);
+    await this.page!.goto(this.pageUrl!);
+  };
+
+  expectSelector = async (config: ExpectSelectorConfig): Promise<void> => {
+    this.throwIfNotInitialized();
+
+    const { selector, containsText, isVisible } = config;
+    const visible = isVisible || true;
+    const text = containsText;
+    const options = { visible, text } as any;
+    await expect(this.page).toMatchElement(selector, options);
+  };
+
+  waitForResponse = async (): Promise<void> => {
+    this.throwIfNotInitialized();
+
+    await this.page!.waitForResponse(response => response.url() === this.pageUrl && response.status() === 200);
+  };
+
+  waitForNavigation = async (): Promise<void> => {
+    this.throwIfNotInitialized();
+
+    await this.page!.waitForNavigation();
+  };
+
+  getUrl = async (): Promise<string> => {
+    this.throwIfNotInitialized();
+
+    return await this.page!.url();
+  };
+
+  getUrlWithoutBaseUrl = async (): Promise<string> => {
+    this.throwIfNotInitialized();
+
+    const url = await this.getUrl();
+
+    return url.replace(constants.baseUrl, '');
+  };
+
+  waitFor = async (milliseconds: number) => {
+    this.throwIfNotInitialized();
+
+    await this.page!.waitFor(milliseconds);
+  };
+
+  private throwIfNotInitialized = () => {
+    if (!this.page) {
+      throw new Error('pageFactory has not been initilized, did you forget to call init with a page?');
+    }
+  };
+}

packages/grafana-toolkit/src/e2e/scenario.ts

@@ -0,0 +1,30 @@
+import { Browser, Page } from 'puppeteer-core';
+import { launchBrowser } from './launcher';
+import { ensureLoggedIn } from './login';
+
+export const e2eScenario = (
+  title: string,
+  testDescription: string,
+  callback: (browser: Browser, page: Page) => void
+) => {
+  describe(title, () => {
+    let browser: Browser;
+    let page: Page;
+
+    beforeAll(async () => {
+      browser = await launchBrowser();
+      page = await browser.newPage();
+      await ensureLoggedIn(page);
+    });
+
+    afterAll(async () => {
+      if (browser) {
+        await browser.close();
+      }
+    });
+
+    it(testDescription, async () => {
+      await callback(browser, page);
+    });
+  });
+};

packages/grafana-toolkit/src/e2e/start/loginPage.ts

@@ -0,0 +1,23 @@
+import { TestPage } from '../pages';
+import {
+  Selector,
+  InputPageObject,
+  InputPageObjectType,
+  ClickablePageObjectType,
+  ClickablePageObject,
+} from '../pageObjects';
+
+export interface LoginPage {
+  username: InputPageObjectType;
+  password: InputPageObjectType;
+  submit: ClickablePageObjectType;
+}
+
+export const loginPage = new TestPage<LoginPage>({
+  url: '/login',
+  pageObjects: {
+    username: new InputPageObject(Selector.fromAriaLabel('Username input field')),
+    password: new InputPageObject(Selector.fromAriaLabel('Password input field')),
+    submit: new ClickablePageObject(Selector.fromAriaLabel('Login button')),
+  },
+});

packages/grafana-toolkit/src/index.ts

@@ -0,0 +1 @@
+export * from './e2e';