@grafana/e2e: screenshots and panel flow (#25203)

* Cleanup

* addPanel now supports (optional) custom dashboardUid

* addPanel now supports (optional) visualization name

* Added CLI option for updating screenshot fixtures

* Added support for console.* functions within tests

* Refactored screenshot command for greater simplicity

* addPanel now sets a unique title

* Updated lockfile

packages/grafana-e2e/cli.js

@@ -3,12 +3,18 @@ const program = require('commander');
 const resolveBin = require('resolve-as-bin');
 const { resolve, sep } = require('path');
 
-const cypress = commandName => {
+const cypress = (commandName, { updateScreenshots }) => {
   // Support running an unpublished dev build
   const dirname = __dirname.split(sep).pop();
   const projectPath = resolve(`${__dirname}${dirname === 'dist' ? '/..' : ''}`);
 
-  const cypressOptions = [commandName, '--env', `CWD=${process.cwd()}`, `--project=${projectPath}`];
+  // For plugins/extendConfig
+  const CWD = `CWD=${process.cwd()}`;
+
+  // For plugins/compareSnapshots
+  const UPDATE_SCREENSHOTS = `UPDATE_SCREENSHOTS=${updateScreenshots ? 1 : 0}`;
+
+  const cypressOptions = [commandName, '--env', `${CWD},${UPDATE_SCREENSHOTS}`, `--project=${projectPath}`];
 
   const execaOptions = {
     cwd: __dirname,
@@ -24,20 +30,20 @@ const cypress = commandName => {
 };
 
 module.exports = () => {
-  const configOption = '-c, --config <path>';
+  const updateOption = '-u, --update-screenshots';
-  const configDescription = 'path to JSON file where configuration values are set; defaults to "cypress.json"';
+  const updateDescription = 'update expected screenshots';
 
   program
     .command('open')
     .description('runs tests within the interactive GUI')
-    .option(configOption, configDescription)
+    .option(updateOption, updateDescription)
-    .action(() => cypress('open'));
+    .action(options => cypress('open', options));
 
   program
     .command('run')
     .description('runs tests from the CLI without the GUI')
-    .option(configOption, configDescription)
+    .option(updateOption, updateDescription)
-    .action(() => cypress('run'));
+    .action(options => cypress('run', options));
 
   program.parse(process.argv);
 };

packages/grafana-e2e/cypress/fixtures/example.json

@@ -1,5 +0,0 @@
-{
-  "name": "Using fixtures to represent data",
-  "email": "hello@cypress.io",
-  "body": "Fixtures are a great way to mock data for responses to routes"
-}

packages/grafana-e2e/cypress/plugins/compareScreenshots.js

@@ -0,0 +1,49 @@
+'use strict';
+const BlinkDiff = require('blink-diff');
+const { resolve } = require('path');
+
+// @todo use npmjs.com/pixelmatch or an available cypress plugin
+const compareSceenshots = async ({ config, screenshotsFolder, specName }) => {
+  const name = config.name || config; // @todo use `??`
+  const threshold = config.threshold || 0.001; // @todo use `??`
+
+  const imageAPath = `${screenshotsFolder}/${specName}/${name}.png`;
+  const imageBPath = resolve(`${screenshotsFolder}/../expected/${specName}/${name}.png`);
+
+  const imageOutputPath = screenshotsFolder.endsWith('actual') ? imageAPath.replace('.png', '.diff.png') : undefined;
+
+  const { code } = await new Promise((resolve, reject) => {
+    new BlinkDiff({
+      imageAPath,
+      imageBPath,
+      imageOutputPath,
+      threshold,
+      thresholdType: BlinkDiff.THRESHOLD_PERCENT,
+    }).run((error, result) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve(result);
+      }
+    });
+  });
+
+  if (code <= 1) {
+    let msg = `\nThe screenshot [${imageAPath}] differs from [${imageBPath}]`;
+    msg += '\n';
+    msg += '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.';
+    msg += '\n';
+    msg += '\n  If the difference between expected and outcome is NOT acceptable then do the following:';
+    msg += '\n    - Check the code for changes that causes this difference, fix that and retry.';
+    msg += '\n';
+    msg += '\n  If the difference between expected and outcome is acceptable then do the following:';
+    msg += '\n    - Replace the expected image with the outcome and retry.';
+    msg += '\n';
+    throw new Error(msg);
+  } else {
+    // Must return a value
+    return true;
+  }
+};
+
+module.exports = compareSceenshots;

packages/grafana-e2e/cypress/plugins/compareSnapshots.js

@@ -1,25 +0,0 @@
-const BlinkDiff = require('blink-diff');
-
-function compareSnapshotsPlugin(args) {
-  args.threshold = args.threshold || 0.001;
-
-  return new Promise((resolve, reject) => {
-    const diff = new BlinkDiff({
-      imageAPath: args.pathToFileA,
-      imageBPath: args.pathToFileB,
-      thresholdType: BlinkDiff.THRESHOLD_PERCENT,
-      threshold: args.threshold,
-      imageOutputPath: args.pathToFileA.replace('.png', '.diff.png'),
-    });
-
-    diff.run((error, result) => {
-      if (error) {
-        reject(error);
-      } else {
-        resolve(result);
-      }
-    });
-  });
-}
-
-module.exports = compareSnapshotsPlugin;

packages/grafana-e2e/cypress/plugins/extendConfig.js

@@ -4,10 +4,11 @@ const {
 } = require('fs');
 const { resolve } = require('path');
 
+// @todo use https://github.com/bahmutov/cypress-extends when possible
 module.exports = async baseConfig => {
   // From CLI
   const {
-    env: { CWD },
+    env: { CWD, UPDATE_SCREENSHOTS },
   } = baseConfig;
 
   if (CWD) {
@@ -21,7 +22,7 @@ module.exports = async baseConfig => {
       reporterOptions: {
         output: `${CWD}/cypress/report.json`,
       },
-      screenshotsFolder: `${CWD}/cypress/screenshots`,
+      screenshotsFolder: `${CWD}/cypress/screenshots/${UPDATE_SCREENSHOTS ? 'expected' : 'actual'}`,
       videosFolder: `${CWD}/cypress/videos`,
     };
 

packages/grafana-e2e/cypress/plugins/index.js

@@ -1,24 +1,22 @@
-const compareSnapshotsPlugin = require('./compareSnapshots');
+const compareScreenshots = require('./compareScreenshots');
 const extendConfig = require('./extendConfig');
 const readProvisions = require('./readProvisions');
 const typescriptPreprocessor = require('./typescriptPreprocessor');
+const { install: installConsoleLogger } = require('cypress-log-to-output');
 
 module.exports = (on, config) => {
-  // yarn build fails with:
-  // >> /Users/hugo/go/src/github.com/grafana/grafana/node_modules/stringmap/stringmap.js:99
-  // >>             throw new Error("StringMap expected string key");
-  // on('task', {
-  //   failed: require('cypress-failed-log/src/failed')(),
-  // });
   on('file:preprocessor', typescriptPreprocessor);
-  on('task', { compareSnapshotsPlugin, readProvisions });
+  on('task', { compareScreenshots, readProvisions });
   on('task', {
+    // @todo remove
     log({ message, optional }) {
       optional ? console.log(message, optional) : console.log(message);
       return null;
     },
   });
 
+  installConsoleLogger(on);
+
   // Always extend with this library's config and return for diffing
   // @todo remove this when possible: https://github.com/cypress-io/cypress/issues/5674
   return extendConfig(config);

packages/grafana-e2e/cypress/support/commands.ts

@@ -1,27 +1,17 @@
-interface CompareSnapshotArgs {
+interface CompareSceenshotsConfig {
-  pathToFileA: string;
+  name: string;
-  pathToFileB: string;
   threshold?: number;
 }
 
-Cypress.Commands.add('compareSnapshot', (args: CompareSnapshotArgs) => {
+Cypress.Commands.add('compareSceenshots', (config: CompareSceenshotsConfig | string) => {
-  cy.task('compareSnapshotsPlugin', args).then((results: any) => {
+  cy.task('compareSceenshots', {
-    if (results.code <= 1) {
+    config,
-      let msg = `\nThe screenshot:[${args.pathToFileA}] differs from :[${args.pathToFileB}]`;
+    screenshotsFolder: Cypress.config('screenshotsFolder'),
-      msg += '\n';
+    specName: Cypress.spec.name,
-      msg += '\nCheck the Artifacts tab in the CircleCi build output for the actual screenshots.';
-      msg += '\n';
-      msg += '\n  If the difference between expected and outcome is NOT acceptable then do the following:';
-      msg += '\n    - Check the code for changes that causes this difference, fix that and retry.';
-      msg += '\n';
-      msg += '\n  If the difference between expected and outcome is acceptable then do the following:';
-      msg += '\n    - Replace the expected image with the outcome and retry.';
-      msg += '\n';
-      throw new Error(msg);
-    }
   });
 });
 
+// @todo remove
 Cypress.Commands.add('logToConsole', (message: string, optional?: any) => {
   cy.task('log', { message, optional });
 });

packages/grafana-e2e/package.json

@@ -50,6 +50,7 @@
     "blink-diff": "1.0.13",
     "commander": "5.0.0",
     "cypress": "4.5.0",
+    "cypress-log-to-output": "^1.0.8",
     "execa": "4.0.0",
     "resolve-as-bin": "2.1.0",
     "ts-loader": "6.2.1",

packages/grafana-e2e/src/flows/addPanel.ts

@@ -2,27 +2,57 @@ import { e2e } from '../index';
 import { getScenarioContext } from '../support/scenarioContext';
 
 export interface AddPanelConfig {
+  dashboardUid?: string;
   dataSourceName: string;
   queriesForm: Function;
+  visualizationName: string;
 }
 
 const DEFAULT_ADD_PANEL_CONFIG: AddPanelConfig = {
   dataSourceName: 'TestData DB',
   queriesForm: () => {},
+  visualizationName: 'Graph',
 };
 
-export const addPanel = (config?: Partial<AddPanelConfig>) => {
+// @todo this actually returns type `Cypress.Chainable`
-  const { dataSourceName, queriesForm } = { ...DEFAULT_ADD_PANEL_CONFIG, ...config };
+export const addPanel = (config?: Partial<AddPanelConfig>): any => {
+  const { dashboardUid, dataSourceName, queriesForm, visualizationName } = { ...DEFAULT_ADD_PANEL_CONFIG, ...config };
+  const panelTitle = `e2e-${Date.now()}`;
 
-  getScenarioContext().then(({ lastAddedDashboardUid }: any) => {
+  return getScenarioContext()
-    e2e.flows.openDashboard(lastAddedDashboardUid);
+    .then(({ lastAddedDashboardUid }: any) => {
-    e2e.pages.Dashboard.Toolbar.toolbarItems('Add panel').click();
+      e2e.flows.openDashboard(dashboardUid ?? lastAddedDashboardUid);
-    e2e.pages.AddDashboard.addNewPanel().click();
+      e2e.pages.Dashboard.Toolbar.toolbarItems('Add panel').click();
-    e2e()
+      e2e.pages.AddDashboard.addNewPanel().click();
-      .get('.ds-picker')
+
-      .click()
+      e2e()
-      .contains('[id^="react-select-"][id*="-option-"]', dataSourceName)
+        .get('.ds-picker')
-      .click();
+        .click()
-    queriesForm();
+        .contains('[id^="react-select-"][id*="-option-"]', dataSourceName)
-  });
+        .click();
+
+      getOptionsGroup('settings')
+        .find('[value="Panel Title"]')
+        .clear()
+        .type(panelTitle);
+      toggleOptionsGroup('settings');
+
+      toggleOptionsGroup('type');
+      e2e()
+        .get(`[aria-label="Plugin visualization item ${visualizationName}"]`)
+        .scrollIntoView()
+        .click();
+      toggleOptionsGroup('type');
+
+      queriesForm();
+    })
+    .then(() => panelTitle);
 };
+
+const getOptionsGroup = (name: string) => e2e().get(`.options-group:has([aria-label="Options group Panel ${name}"])`);
+
+const toggleOptionsGroup = (name: string) =>
+  getOptionsGroup(name)
+    .find('.editor-options-group-toggle')
+    .scrollIntoView()
+    .click();

yarn.lock

@@ -8912,6 +8912,14 @@ chownr@^1.1.1, chownr@^1.1.2:
   resolved "https://registry.yarnpkg.com/chownr/-/chownr-1.1.3.tgz#42d837d5239688d55f303003a508230fa6727142"
   integrity sha512-i70fVHhmV3DtTl6nqvZOnIjbY0Pe4kAUjwHj8z0zAdgBtYrJyYwLKCCuRBQ5ppkyL0AkN7HKRnETdmdp1zqNXw==
 
+chrome-remote-interface@^0.27.1:
+  version "0.27.2"
+  resolved "https://registry.yarnpkg.com/chrome-remote-interface/-/chrome-remote-interface-0.27.2.tgz#e5605605f092b7ef8575d95304e004039c9d0ab9"
+  integrity sha512-pVLljQ29SAx8KIv5tSa9sIf8GrEsAZdPJoeWOmY3/nrIzFmE+EryNNHvDkddGod0cmAFTv+GmPG0uvzxi2NWsA==
+  dependencies:
+    commander "2.11.x"
+    ws "^6.1.0"
+
 chrome-trace-event@^1.0.2:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/chrome-trace-event/-/chrome-trace-event-1.0.2.tgz#234090ee97c7d4ad1a2c4beae27505deffc608a4"
@@ -9301,6 +9309,11 @@ commander@2, commander@^2.18.0, commander@^2.19.0, commander@^2.20.0, commander@
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
   integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
 
+commander@2.11.x:
+  version "2.11.0"
+  resolved "https://registry.yarnpkg.com/commander/-/commander-2.11.0.tgz#157152fd1e7a6c8d98a5b715cf376df928004563"
+  integrity sha512-b0553uYA5YAEGgyYIGYROzKQ7X5RAqedkfjiZxwi0kL1g3bOaBNNZfYkzt/CL0umgD5wc9Jec2FbB98CjkMRvQ==
+
 commander@2.17.x:
   version "2.17.1"
   resolved "https://registry.yarnpkg.com/commander/-/commander-2.17.1.tgz#bd77ab7de6de94205ceacc72f1716d29f20a77bf"
@@ -10201,6 +10214,14 @@ cyclist@^1.0.1:
   resolved "https://registry.yarnpkg.com/cyclist/-/cyclist-1.0.1.tgz#596e9698fd0c80e12038c2b82d6eb1b35b6224d9"
   integrity sha1-WW6WmP0MgOEgOMK4LW6xs1tiJNk=
 
+cypress-log-to-output@^1.0.8:
+  version "1.0.8"
+  resolved "https://registry.yarnpkg.com/cypress-log-to-output/-/cypress-log-to-output-1.0.8.tgz#8698db2cd68b88fd62e7f9ea8d1eece20bb210cf"
+  integrity sha512-o0PwNSXSZho2QLTOa4I/KgyPfZwgqNBqNcz+jBMcKJHPsRBZDUElaosigqSXI28uuSlprUlvcYjpcb/791u/lg==
+  dependencies:
+    chalk "^2.4.2"
+    chrome-remote-interface "^0.27.1"
+
 cypress@4.5.0:
   version "4.5.0"
   resolved "https://registry.yarnpkg.com/cypress/-/cypress-4.5.0.tgz#01940d085f6429cec3c87d290daa47bb976a7c7b"