mirror of
https://github.com/grafana/grafana.git
synced 2026-08-19 01:34:54 -05:00
CI: Fix pa11y check by running in docker-puppeteer image (#107084)
* Change pa11y dagger to run in docker-puppeteer image * export json results * re-enable pa11y ci check * update gha workflow to new flags * add no-threshold-fail, use single pa11y config * fix codeowners * readme * fix drone config
This commit is contained in:
+5
-3
@@ -393,13 +393,14 @@ steps:
|
||||
- packages/grafana-ui/**
|
||||
- commands:
|
||||
- npx wait-on@7.0.1 http://$HOST:$PORT
|
||||
- pa11y-ci --config .pa11yci-pr.conf.js
|
||||
- pa11y-ci --config e2e/pa11yci.conf.js
|
||||
depends_on:
|
||||
- grafana-server
|
||||
environment:
|
||||
GRAFANA_MISC_STATS_API_KEY:
|
||||
from_secret: grafana_misc_stats_api_key
|
||||
HOST: grafana-server
|
||||
NO_THRESHOLDS: "false"
|
||||
PORT: 3001
|
||||
failure: always
|
||||
image: grafana/docker-puppeteer:1.1.0
|
||||
@@ -864,13 +865,14 @@ steps:
|
||||
- packages/grafana-ui/**
|
||||
- commands:
|
||||
- npx wait-on@7.0.1 http://$HOST:$PORT
|
||||
- pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json
|
||||
- pa11y-ci --config e2e/pa11yci.conf.js
|
||||
depends_on:
|
||||
- grafana-server
|
||||
environment:
|
||||
GRAFANA_MISC_STATS_API_KEY:
|
||||
from_secret: grafana_misc_stats_api_key
|
||||
HOST: grafana-server
|
||||
NO_THRESHOLDS: "true"
|
||||
PORT: 3001
|
||||
failure: ignore
|
||||
image: grafana/docker-puppeteer:1.1.0
|
||||
@@ -2984,6 +2986,6 @@ kind: secret
|
||||
name: gcr_credentials
|
||||
---
|
||||
kind: signature
|
||||
hmac: db5485c680a21c2f4032064cd7fd892a3075ebd92cd29422d892fcc375eb6948
|
||||
hmac: 1198b1489e48a9ced211633a0325d112814553246847fc7320fb5ac2bcb32b7d
|
||||
|
||||
...
|
||||
|
||||
@@ -646,8 +646,6 @@ playwright.config.ts @grafana/plugins-platform-frontend
|
||||
/scripts/**/generate-transformations* @grafana/datapro
|
||||
/scripts/webpack/ @grafana/frontend-ops
|
||||
/scripts/generate-a11y-report.sh @grafana/grafana-frontend-platform
|
||||
.pa11yci.conf.js @grafana/grafana-frontend-platform
|
||||
.pa11yci-pr.conf.js @grafana/grafana-frontend-platform
|
||||
.betterer.results @grafanabot
|
||||
.betterer.ts @grafana/grafana-frontend-platform
|
||||
|
||||
|
||||
@@ -137,42 +137,33 @@ jobs:
|
||||
path: videos
|
||||
retention-days: 1
|
||||
|
||||
# Skipping this for now while we fix it
|
||||
# run-a11y-test:
|
||||
# needs:
|
||||
# - build-grafana
|
||||
# - build-e2e-runner
|
||||
# name: A11y test
|
||||
# runs-on: ubuntu-latest-8-cores
|
||||
# permissions:
|
||||
# contents: read
|
||||
run-a11y-test:
|
||||
needs:
|
||||
- build-grafana
|
||||
name: A11y test
|
||||
runs-on: ubuntu-latest-8-cores
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# steps:
|
||||
# - uses: actions/checkout@v4
|
||||
# with:
|
||||
# persist-credentials: false
|
||||
# - uses: actions/download-artifact@v4
|
||||
# with:
|
||||
# name: ${{ needs.build-grafana.outputs.artifact }}
|
||||
# - uses: actions/download-artifact@v4
|
||||
# with:
|
||||
# name: ${{ needs.build-e2e-runner.outputs.artifact }}
|
||||
# - name: chmod +x
|
||||
# run: chmod +x ./e2e-runner
|
||||
# - name: Run PR a11y test
|
||||
# if: github.event_name == 'pull_request'
|
||||
# uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
|
||||
# with:
|
||||
# verb: run
|
||||
# args: go run ./pkg/build/a11y --package=grafana.tar.gz
|
||||
# --flags="--json --config ./.pa11yci-pr.conf.js"
|
||||
# - name: Run non-PR a11y test
|
||||
# if: github.event_name != 'pull_request'
|
||||
# uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
|
||||
# with:
|
||||
# verb: run
|
||||
# args: go run ./pkg/build/a11y --package=grafana.tar.gz
|
||||
# --flags="--json --config ./.pa11yci.conf.js"
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: ${{ needs.build-grafana.outputs.artifact }}
|
||||
- name: Run PR a11y test
|
||||
if: github.event_name == 'pull_request'
|
||||
uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
|
||||
with:
|
||||
verb: run
|
||||
args: go run ./pkg/build/a11y --package=grafana.tar.gz
|
||||
- name: Run non-PR a11y test
|
||||
if: github.event_name != 'pull_request'
|
||||
uses: dagger/dagger-for-github@e47aba410ef9bb9ed81a4d2a97df31061e5e842e
|
||||
with:
|
||||
verb: run
|
||||
args: go run ./pkg/build/a11y --package=grafana.tar.gz --no-threshold-fail
|
||||
|
||||
# This is the job that is actually required by rulesets.
|
||||
# We want to only require one job instead of all the individual tests.
|
||||
|
||||
@@ -1,140 +0,0 @@
|
||||
var dashboardSettings = [
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=settings',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=annotations',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=templating',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=links',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=versions',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=permissions',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=dashboard_json',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
];
|
||||
var config = {
|
||||
defaults: {
|
||||
concurrency: 1,
|
||||
runners: ['axe'],
|
||||
useIncognitoBrowserContext: false,
|
||||
standard: 'WCAG2AA',
|
||||
chromeLaunchConfig: {
|
||||
args: ['--no-sandbox'],
|
||||
},
|
||||
// see https://github.com/grafana/grafana/pull/41693#issuecomment-979921463 for context
|
||||
// on why we're ignoring singleValue/react-select-*-placeholder elements
|
||||
hideElements: '#updateVersion, [class*="-singleValue"], [id^="react-select-"][id$="-placeholder"]',
|
||||
},
|
||||
|
||||
urls: [
|
||||
{
|
||||
url: '${HOST}/login',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/login', //skip password and login
|
||||
actions: [
|
||||
"wait for element input[name='user'] to be added",
|
||||
"set field input[name='user'] to admin",
|
||||
"set field input[name='password'] to admin",
|
||||
"click element button[data-testid='data-testid Login button']",
|
||||
"wait for element button[data-testid='data-testid Skip change password button'] to be visible",
|
||||
],
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/?orgId=1',
|
||||
wait: 500,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
...dashboardSettings,
|
||||
{
|
||||
url: '${HOST}/?orgId=1&search=open',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/alerting/list',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/datasources',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/org/users',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/org/teams',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/plugins',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/org',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/org/apikeys',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/dashboards',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
function myPa11yCiConfiguration(urls, defaults) {
|
||||
const HOST_SERVER = process.env.HOST || 'localhost';
|
||||
const PORT_SERVER = process.env.PORT || '3001';
|
||||
for (var idx = 0; idx < urls.length; idx++) {
|
||||
urls[idx] = { ...urls[idx], url: urls[idx].url.replace('${HOST}', `${HOST_SERVER}:${PORT_SERVER}`) };
|
||||
}
|
||||
return {
|
||||
defaults: defaults,
|
||||
urls: urls,
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = myPa11yCiConfiguration(config.urls, config.defaults);
|
||||
@@ -1,50 +1,4 @@
|
||||
var dashboardSettings = [
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=settings',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=annotations',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=templating',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=links',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=versions',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=permissions',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
// TODO: improve the accessibility of the permission tab https://github.com/grafana/grafana/issues/77203
|
||||
threshold: 5,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=dashboard_json',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 2,
|
||||
},
|
||||
];
|
||||
|
||||
var config = {
|
||||
const config = {
|
||||
defaults: {
|
||||
concurrency: 1,
|
||||
runners: ['axe'],
|
||||
@@ -57,18 +11,16 @@ var config = {
|
||||
// see https://github.com/grafana/grafana/pull/41693#issuecomment-979921463 for context
|
||||
// on why we're ignoring singleValue/react-select-*-placeholder elements
|
||||
hideElements: '#updateVersion, [class*="-singleValue"], [id^="react-select-"][id$="-placeholder"]',
|
||||
reporters: ['cli', ['json', { fileName: './pa11y-ci-results.json' }]],
|
||||
},
|
||||
|
||||
urls: [
|
||||
{
|
||||
url: '${HOST}/login',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/login',
|
||||
wait: 500,
|
||||
actions: [
|
||||
"wait for element input[name='user'] to be added",
|
||||
"set field input[name='user'] to admin",
|
||||
@@ -77,74 +29,84 @@ var config = {
|
||||
"wait for element button[data-testid='data-testid Skip change password button'] to be visible",
|
||||
],
|
||||
threshold: 2,
|
||||
rootElement: '.main-view',
|
||||
},
|
||||
{
|
||||
url: '${HOST}/?orgId=1',
|
||||
wait: 500,
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
...dashboardSettings,
|
||||
|
||||
// Dashboard settings
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=settings',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=annotations',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=variables',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=links',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=versions',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=permissions',
|
||||
// TODO: improve the accessibility of the permission tab https://github.com/grafana/grafana/issues/77203
|
||||
threshold: 5,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=dashboard_json',
|
||||
threshold: 2,
|
||||
},
|
||||
|
||||
// Misc
|
||||
{
|
||||
url: '${HOST}/?orgId=1&search=open',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/alerting/list',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
// the unified alerting promotion alert's content contrast is too low
|
||||
// see https://github.com/grafana/grafana/pull/41829
|
||||
threshold: 7,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/datasources',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/org/users',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 2,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/org/teams',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/plugins',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 0,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/org',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 2,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/org/apikeys',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 4,
|
||||
},
|
||||
{
|
||||
url: '${HOST}/dashboards',
|
||||
wait: 500,
|
||||
rootElement: '.main-view',
|
||||
threshold: 2,
|
||||
},
|
||||
],
|
||||
@@ -153,9 +115,21 @@ var config = {
|
||||
function myPa11yCiConfiguration(urls, defaults) {
|
||||
const HOST_SERVER = process.env.HOST || 'localhost';
|
||||
const PORT_SERVER = process.env.PORT || '3001';
|
||||
for (var idx = 0; idx < urls.length; idx++) {
|
||||
urls[idx] = { ...urls[idx], url: urls[idx].url.replace('${HOST}', `${HOST_SERVER}:${PORT_SERVER}`) };
|
||||
}
|
||||
const noThresholds = process.env.NO_THRESHOLDS === 'true';
|
||||
|
||||
urls = urls.map((test, index) => {
|
||||
return {
|
||||
...test,
|
||||
url: test.url.replace('${HOST}', `${HOST_SERVER}:${PORT_SERVER}`),
|
||||
screenCapture: `./screenshots/screenshot-${index}.png`,
|
||||
rootElement: '.main-view',
|
||||
wait: 500,
|
||||
|
||||
// Depending on NO_THRESHOLDS (--no-threshold-fail in the dagger command), clear the thresholds
|
||||
// to allow pa11y to fail the check and include error details in the results file
|
||||
threshold: noThresholds ? undefined : test.threshold,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
defaults: defaults,
|
||||
@@ -0,0 +1,22 @@
|
||||
# Pa11y accessability tests
|
||||
|
||||
We use pa11y to run some automated simple accessability tests. They're ran with dagger to help orchestrate starting server + tests in a reproducable manner.
|
||||
|
||||
To run the tests locally:
|
||||
|
||||
1. Install dagger locally https://docs.dagger.io/install/
|
||||
2. Grab the grafana.tar.gz artifact by either
|
||||
1. Downloading it from the Github Action artifact from your PR
|
||||
1. Build it locally with:
|
||||
```sh
|
||||
dagger run go run ./pkg/build/cmd artifacts -a targz:grafana:linux/amd64 --grafana-dir="$PWD" > dist/files.txt
|
||||
cat dist/files.txt # Will output the path to the grafana.tar.gz
|
||||
```
|
||||
3. Run the dagger pipeline with:
|
||||
```sh
|
||||
dagger -v run go run ./pkg/build/a11y --package=(full path to .tar.gz) --results=./pa11y-ci-results.json
|
||||
```
|
||||
The JSON results file will be saved to the file from the `--results` arg
|
||||
4. If they fail and you want to see the full output
|
||||
1. Run the dagger command with `dagger -vE [...]`
|
||||
2. At the end, arrow up to the exec pa11y-ci segment and hit Enter
|
||||
+53
-26
@@ -6,12 +6,16 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
|
||||
"dagger.io/dagger"
|
||||
"github.com/urfave/cli/v3"
|
||||
)
|
||||
|
||||
var (
|
||||
grafanaHost = "grafana"
|
||||
grafanaPort = 3001
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt)
|
||||
defer cancel()
|
||||
@@ -49,8 +53,21 @@ func NewApp() *cli.Command {
|
||||
TakesFile: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "flags",
|
||||
Usage: "Flags to pass through to the e2e runner",
|
||||
Name: "config",
|
||||
Usage: "Path to the pa11y config file to use",
|
||||
Value: "e2e/pa11yci.conf.js",
|
||||
Validator: mustBeFile("config", true),
|
||||
TakesFile: true,
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "results",
|
||||
Usage: "Path to the pa11y results file to export",
|
||||
TakesFile: true,
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "no-threshold-fail",
|
||||
Usage: "Don't fail the task if any of the tests fail. Use this in combination with --results to list all violations even if they're within thresholds",
|
||||
Value: false,
|
||||
},
|
||||
},
|
||||
Action: run,
|
||||
@@ -61,26 +78,29 @@ func run(ctx context.Context, cmd *cli.Command) error {
|
||||
grafanaDir := cmd.String("grafana-dir")
|
||||
targzPath := cmd.String("package")
|
||||
licensePath := cmd.String("license")
|
||||
runnerFlags := cmd.String("flags")
|
||||
pa11yConfigPath := cmd.String("config")
|
||||
pa11yResultsPath := cmd.String("results")
|
||||
noThresholdFail := cmd.Bool("no-threshold-fail")
|
||||
|
||||
d, err := dagger.Connect(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to connect to Dagger: %w", err)
|
||||
}
|
||||
|
||||
yarnCache := d.CacheVolume("yarn")
|
||||
|
||||
//nolint:gosec
|
||||
nvmrcContents, err := os.ReadFile(filepath.Join(grafanaDir, ".nvmrc"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to read .nvmrc file: %w", err)
|
||||
}
|
||||
nodeVersion := string(nvmrcContents)
|
||||
|
||||
grafana := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{
|
||||
Exclude: []string{"node_modules", "*.tar.gz"},
|
||||
// Explicitly only the files used by the grafana-server service
|
||||
hostSrc := d.Host().Directory(grafanaDir, dagger.HostDirectoryOpts{
|
||||
Include: []string{
|
||||
"./devenv",
|
||||
"./e2e/test-plugins", // Directory is included so provisioning works, but they're not actually build
|
||||
"./scripts/grafana-server/custom.ini",
|
||||
"./scripts/grafana-server/start-server",
|
||||
"./scripts/grafana-server/kill-server",
|
||||
"./scripts/grafana-server/variables",
|
||||
},
|
||||
})
|
||||
|
||||
targz := d.Host().File(targzPath)
|
||||
pa11yConfig := d.Host().File(pa11yConfigPath)
|
||||
|
||||
var license *dagger.File
|
||||
if licensePath != "" {
|
||||
@@ -88,27 +108,34 @@ func run(ctx context.Context, cmd *cli.Command) error {
|
||||
}
|
||||
|
||||
svc, err := GrafanaService(ctx, d, GrafanaServiceOpts{
|
||||
GrafanaDir: grafana,
|
||||
HostSrc: hostSrc,
|
||||
GrafanaTarGz: targz,
|
||||
License: license,
|
||||
YarnCache: yarnCache,
|
||||
NodeVersion: nodeVersion,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create Grafana service: %w", err)
|
||||
}
|
||||
|
||||
c := RunTest(d, svc, grafana, yarnCache, nodeVersion, runnerFlags)
|
||||
c, err = c.Sync(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to run a11y test suite: %w", err)
|
||||
c, runErr := RunTest(ctx, d, svc, pa11yConfig, noThresholdFail, pa11yResultsPath)
|
||||
if runErr != nil {
|
||||
return fmt.Errorf("failed to run a11y test suite: %w", runErr)
|
||||
}
|
||||
|
||||
code, err := c.ExitCode(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get exit code of a11y test suite: %w", err)
|
||||
c, syncErr := c.Sync(ctx)
|
||||
if syncErr != nil {
|
||||
return fmt.Errorf("failed to sync a11y test suite: %w", syncErr)
|
||||
}
|
||||
if code != 0 {
|
||||
|
||||
code, codeErr := c.ExitCode(ctx)
|
||||
if codeErr != nil {
|
||||
return fmt.Errorf("failed to get exit code of a11y test suite: %w", codeErr)
|
||||
}
|
||||
|
||||
if code == 0 {
|
||||
log.Printf("a11y tests passed with exit code %d", code)
|
||||
} else if noThresholdFail {
|
||||
log.Printf("a11y tests failed with exit code %d, but noFail is true", code)
|
||||
} else {
|
||||
return fmt.Errorf("a11y tests failed with exit code %d", code)
|
||||
}
|
||||
|
||||
|
||||
+36
-13
@@ -1,25 +1,48 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"dagger.io/dagger"
|
||||
)
|
||||
|
||||
func RunTest(
|
||||
ctx context.Context,
|
||||
d *dagger.Client,
|
||||
svc *dagger.Service,
|
||||
src *dagger.Directory, cache *dagger.CacheVolume,
|
||||
nodeVersion, runnerFlags string) *dagger.Container {
|
||||
command := fmt.Sprintf(
|
||||
"./e2e-runner a11y --start-grafana=false"+
|
||||
" --grafana-host grafana --grafana-port 3001 %s", runnerFlags)
|
||||
|
||||
return GrafanaFrontend(d, cache, nodeVersion, src).
|
||||
WithExec([]string{"/bin/sh", "-c", "apt-get update && apt-get install -y git curl"}).
|
||||
WithExec([]string{"curl", "-LO", "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb"}).
|
||||
WithExec([]string{"apt-get", "install", "-y", "./google-chrome-stable_current_amd64.deb"}).
|
||||
grafanaService *dagger.Service,
|
||||
pa11yConfig *dagger.File,
|
||||
noThresholdFail bool,
|
||||
pa11yResultsPath string,
|
||||
) (*dagger.Container, error) {
|
||||
// docker-puppeteer container already has Chrome and Pa11y installed in it
|
||||
pa11yContainer := d.Container().From("grafana/docker-puppeteer:1.1.0").
|
||||
WithWorkdir("/src").
|
||||
WithServiceBinding("grafana", svc).
|
||||
WithExec([]string{"/bin/bash", "-c", command}, dagger.ContainerWithExecOpts{Expect: dagger.ReturnTypeAny})
|
||||
WithExec([]string{"mkdir", "-p", "./screenshots"}). // not yet exported
|
||||
WithEnvVariable("HOST", grafanaHost).
|
||||
WithEnvVariable("PORT", fmt.Sprint(grafanaPort))
|
||||
|
||||
if noThresholdFail {
|
||||
// This logic is non-intuitive - --no-threshold-fail will make pa11y fail (by removing thresholds from the config)
|
||||
// so it can write all violations to the results file. This failure is then ignored by the caller in main.go.
|
||||
// Otherwise, pa11y ignores violations if they're within the thresholds and doesn't include them in the results file
|
||||
pa11yContainer = pa11yContainer.
|
||||
WithEnvVariable("NO_THRESHOLDS", "true")
|
||||
}
|
||||
|
||||
pa11yContainer = pa11yContainer.
|
||||
WithServiceBinding(grafanaHost, grafanaService).
|
||||
WithMountedFile("pa11yci-config.js", pa11yConfig).
|
||||
WithExec([]string{"pa11y-ci", "--config", "pa11yci-config.js"}, dagger.ContainerWithExecOpts{
|
||||
Expect: dagger.ReturnTypeAny, // allow this to fail here so we can handle non-zero exit codes at the caller
|
||||
})
|
||||
|
||||
if pa11yResultsPath != "" {
|
||||
_, err := pa11yContainer.File("/src/pa11y-ci-results.json").Export(ctx, pa11yResultsPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get pa11y results: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
return pa11yContainer, nil
|
||||
}
|
||||
|
||||
@@ -9,77 +9,31 @@ import (
|
||||
"dagger.io/dagger"
|
||||
)
|
||||
|
||||
func NodeImage(version string) string {
|
||||
return fmt.Sprintf("node:%s-slim", strings.TrimPrefix(strings.TrimSpace(version), "v"))
|
||||
}
|
||||
|
||||
type GrafanaServiceOpts struct {
|
||||
GrafanaDir *dagger.Directory
|
||||
HostSrc *dagger.Directory
|
||||
GrafanaTarGz *dagger.File
|
||||
License *dagger.File
|
||||
YarnCache *dagger.CacheVolume
|
||||
NodeVersion string
|
||||
}
|
||||
|
||||
func Frontend(src *dagger.Directory) *dagger.Directory {
|
||||
return src.
|
||||
WithoutFile("go.mod").
|
||||
WithoutFile("go.sum").
|
||||
WithoutFile("go.work").
|
||||
WithoutFile("go.work.sum").
|
||||
WithoutDirectory(".github").
|
||||
WithoutDirectory("docs").
|
||||
WithoutDirectory("pkg").
|
||||
WithoutDirectory("apps").
|
||||
WithoutDirectory("videos")
|
||||
}
|
||||
|
||||
func WithGrafanaFrontend(c *dagger.Container, src *dagger.Directory) *dagger.Container {
|
||||
return c.WithDirectory("/src", Frontend(src), dagger.ContainerWithDirectoryOpts{
|
||||
Exclude: []string{
|
||||
"*drone*",
|
||||
"*.go",
|
||||
"*.md",
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
func WithYarnCache(c *dagger.Container, cache *dagger.CacheVolume) *dagger.Container {
|
||||
return c.
|
||||
WithWorkdir("/src").
|
||||
WithMountedCache("/yarn/cache", cache)
|
||||
}
|
||||
|
||||
func GrafanaFrontend(d *dagger.Client, yarnCache *dagger.CacheVolume, nodeVersion string, grafanaDir *dagger.Directory) *dagger.Container {
|
||||
container := d.Container().From(NodeImage(nodeVersion))
|
||||
container = WithGrafanaFrontend(container, grafanaDir)
|
||||
return WithYarnCache(container, yarnCache).
|
||||
WithEnvVariable("YARN_CACHE_FOLDER", "/yarn/cache").
|
||||
WithExec([]string{"yarn", "install", "--immutable"})
|
||||
}
|
||||
|
||||
func GrafanaService(ctx context.Context, d *dagger.Client, opts GrafanaServiceOpts) (*dagger.Service, error) {
|
||||
src := GrafanaFrontend(d, opts.YarnCache, opts.NodeVersion, opts.GrafanaDir)
|
||||
|
||||
container := d.Container().From("alpine:3").
|
||||
WithExec([]string{"apk", "add", "--no-cache", "bash", "tar", "netcat-openbsd"}).
|
||||
WithMountedFile("/src/grafana.tar.gz", opts.GrafanaTarGz).
|
||||
WithExec([]string{"mkdir", "-p", "/src/grafana"}).
|
||||
WithExec([]string{"tar", "--strip-components=1", "-xzf", "/src/grafana.tar.gz", "-C", "/src/grafana"}).
|
||||
WithDirectory("/src/grafana/devenv", src.Directory("/src/devenv")).
|
||||
WithDirectory("/src/grafana/e2e", src.Directory("/src/e2e")).
|
||||
WithDirectory("/src/grafana/scripts", src.Directory("/src/scripts")).
|
||||
WithDirectory("/src/grafana/tools", src.Directory("/src/tools")).
|
||||
WithDirectory("/src/grafana/devenv", opts.HostSrc.Directory("./devenv")).
|
||||
WithDirectory("/src/grafana/e2e/test-plugins", opts.HostSrc.Directory("./e2e/test-plugins")).
|
||||
WithDirectory("/src/grafana/scripts", opts.HostSrc.Directory("./scripts")).
|
||||
WithWorkdir("/src/grafana").
|
||||
WithEnvVariable("GF_APP_MODE", "development").
|
||||
WithEnvVariable("GF_SERVER_HTTP_PORT", "3001").
|
||||
WithEnvVariable("GF_SERVER_HTTP_PORT", fmt.Sprint(grafanaPort)).
|
||||
WithEnvVariable("GF_SERVER_ROUTER_LOGGING", "1").
|
||||
WithExposedPort(3001)
|
||||
WithExposedPort(grafanaPort)
|
||||
|
||||
var licenseArg string
|
||||
if opts.License != nil {
|
||||
container = container.WithMountedFile("/src/license.jwt", opts.License)
|
||||
licenseArg = "/src/license.jwt"
|
||||
container = container.WithMountedFile(licenseArg, opts.License)
|
||||
}
|
||||
|
||||
// We add all GF_ environment variables to allow for overriding Grafana configuration.
|
||||
|
||||
@@ -649,21 +649,13 @@ def test_a11y_frontend_step(ver_mode, port = 3001):
|
||||
commands = [
|
||||
# Note - this runs in a container running node 14, which does not support the -y option to npx
|
||||
"npx wait-on@7.0.1 http://$HOST:$PORT",
|
||||
"pa11y-ci --config e2e/pa11yci.conf.js",
|
||||
]
|
||||
failure = "ignore"
|
||||
no_thresholds = "true"
|
||||
if ver_mode == "pr":
|
||||
commands.extend(
|
||||
[
|
||||
"pa11y-ci --config .pa11yci-pr.conf.js",
|
||||
],
|
||||
)
|
||||
failure = "always"
|
||||
else:
|
||||
commands.extend(
|
||||
[
|
||||
"pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json",
|
||||
],
|
||||
)
|
||||
no_thresholds = "false"
|
||||
|
||||
return {
|
||||
"name": "test-a11y-frontend",
|
||||
@@ -676,6 +668,7 @@ def test_a11y_frontend_step(ver_mode, port = 3001):
|
||||
"GRAFANA_MISC_STATS_API_KEY": from_secret("grafana_misc_stats_api_key"),
|
||||
"HOST": "grafana-server",
|
||||
"PORT": port,
|
||||
"NO_THRESHOLDS": no_thresholds,
|
||||
},
|
||||
"failure": failure,
|
||||
"commands": commands,
|
||||
|
||||
Reference in New Issue
Block a user