From 1edd415ddff6247b1d7d95cc5c48b65c59b172ec Mon Sep 17 00:00:00 2001 From: Maria Alexandra <239999+axelavargas@users.noreply.github.com> Date: Tue, 14 Sep 2021 13:23:17 +0200 Subject: [PATCH] Accessibility checks: Phase 2 - Adding Pa11y CI PR stage (#38556) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Hugo Häggmark --- .drone.yml | 15 ++- .gitignore | 4 + .pa11yci-pr.conf.js | 96 +++++++++++++++++++ contribute/create-pull-request.md | 11 +++ .../datasources/aws-cloudwatch/_index.md | 4 +- package.json | 3 + scripts/drone/pipelines/pr.star | 2 + scripts/drone/steps/lib.star | 18 ++++ scripts/generate-a11y-report.sh | 27 ++++++ yarn.lock | 59 ++++++++++-- 10 files changed, 229 insertions(+), 10 deletions(-) create mode 100644 .pa11yci-pr.conf.js create mode 100755 scripts/generate-a11y-report.sh diff --git a/.drone.yml b/.drone.yml index fdab05e6723..2149ef66724 100644 --- a/.drone.yml +++ b/.drone.yml @@ -146,6 +146,19 @@ steps: depends_on: - package +- name: test-a11y-frontend-pr + image: buildkite/puppeteer + commands: + - yarn wait-on http://$HOST:$PORT + - yarn -s test:accessibility-pr + environment: + GRAFANA_MISC_STATS_API_KEY: + from_secret: grafana_misc_stats_api_key + HOST: end-to-end-tests-server + PORT: 3001 + depends_on: + - end-to-end-tests-server + - name: build-frontend-docs image: grafana/build-container:1.4.2 commands: @@ -3478,6 +3491,6 @@ get: --- kind: signature -hmac: 1ed78e5f23e4e069c2a2cdb34d5220b556a76c812b23fe4bdfad09289363370b +hmac: 502c06a59a21e0d44e32814700f0eb6cd37b8fe2b959b1b87d5ae0dc1658c3c4 ... diff --git a/.gitignore b/.gitignore index 134faa1e7ee..7d62dd7d7a9 100644 --- a/.gitignore +++ b/.gitignore @@ -123,6 +123,10 @@ compilation-stats.json !/e2e/**/screenshots/expected/* /e2e/**/videos/* +# a11y tests +/pa11y-ci-results.json +/pa11y-ci-report + # report dumping the whole system env /report.*.json diff --git a/.pa11yci-pr.conf.js b/.pa11yci-pr.conf.js new file mode 100644 index 00000000000..cc7c6ad64ef --- /dev/null +++ b/.pa11yci-pr.conf.js @@ -0,0 +1,96 @@ +var config = { + defaults: { + concurrency: 1, + runners: ['axe'], + chromeLaunchConfig: { + args: ['--no-sandbox'], + }, + }, + + urls: [ + { + url: '${HOST}/login', + actions: [ + "set field input[name='user'] to admin", + "set field input[name='password'] to admin", + "click element button[aria-label='Login button']", + "wait for element [aria-label='Skip change password button'] to be visible", + ], + threshold: 2, + }, + { + url: '${HOST}/?orgId=1', + threshold: 7, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge', + hideElements: '.sidemenu', + threshold: 2, + }, + { + url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=settings', + rootElement: '.dashboard-settings', + threshold: 10, + }, + { + url: '${HOST}/?orgId=1&search=open', + rootElement: '.main-view', + threshold: 15, + }, + { + url: '${HOST}/alerting/list', + rootElement: '.main-view', + threshold: 7, + }, + { + url: '${HOST}/datasources', + rootElement: '.main-view', + threshold: 36, + }, + { + url: '${HOST}/org/users', + rootElement: '.main-view', + threshold: 4, + }, + { + url: '${HOST}/org/teams', + rootElement: '.main-view', + threshold: 1, + }, + { + url: '${HOST}/plugins', + rootElement: '.main-view', + threshold: 41, + }, + { + url: '${HOST}/org', + rootElement: '.main-view', + threshold: 2, + }, + { + url: '${HOST}/org/apikeys', + rootElement: '.main-view', + threshold: 5, + }, + { + url: '${HOST}/dashboards', + rootElement: '.main-view', + threshold: 8, + }, + ], +}; + +function myPa11yCiConfiguration(urls, defaults) { + const HOST_SERVER = process.env.HOST || 'localhost'; + const PORT_SERVER = process.env.PORT || '3000'; + for (var idx = 0; idx < urls.length; idx++) { + urls[idx] = { ...urls[idx], url: urls[idx].url.replace('${HOST}', `${HOST_SERVER}:${PORT_SERVER}`) }; + } + + return { + defaults: defaults, + urls: urls, + }; +} + +module.exports = myPa11yCiConfiguration(config.urls, config.defaults); diff --git a/contribute/create-pull-request.md b/contribute/create-pull-request.md index 504fdba44e9..b8009c2b94d 100644 --- a/contribute/create-pull-request.md +++ b/contribute/create-pull-request.md @@ -55,6 +55,17 @@ Pull requests that create new UI components or modify existing ones must adhere - Use the [Grafana theme palette](/contribute/style-guides/themes.md) for styling. It contains colors with good contrast which aids accessibility. - Use [RTL](https://testing-library.com/docs/dom-testing-library/api-accessibility/) for writing unit tests. It helps to create accessible components. +Pull requests that introduce accessibility(a11y) errors: + +We use [pa11y-ci](https://github.com/pa11y/pa11y-ci) to collect accessibility errors on [some URLs on the project](https://github.com/grafana/grafana/issues/36555), threshold errors are specified per URL. + +If the contribution introduces new a11y errors, our continuous integration will fail, preventing you to merge on the main branch. In those cases there are two alternatives for moving forward: + +- Check the error log on the pipeline step `test-a11y-frontend-pr`, identify what was the error, and fix it. +- Locally run the command `yarn test:accessibility-report` that generates an HTML accessibility report, then go to the URL that contains your change, identify the error, and fix it. Keep in mind, a local Grafana instance needs to be running on `http://localhost:3000`. + +You can also prevent introducing a11y errors by installing an a11y plugin in your browser, for example, axe DevTools, Accessibility Insights for Web among others. + ### Backend-specific guidelines Please refer to the [backend style guidelines](/contribute/style-guides/backend.md). diff --git a/docs/sources/datasources/aws-cloudwatch/_index.md b/docs/sources/datasources/aws-cloudwatch/_index.md index 01b418b3f3c..635bc61cb2e 100644 --- a/docs/sources/datasources/aws-cloudwatch/_index.md +++ b/docs/sources/datasources/aws-cloudwatch/_index.md @@ -31,7 +31,7 @@ To access data source settings, hover your mouse over the **Configuration** (gea ## Authentication -For authentication options and configuration details, see [AWS authentication]({{< relref "aws-authentication.md" >}}) topic. +For authentication options and configuration details, see [AWS authentication]({{< relref "aws-authentication.md" >}}) topic. ## IAM policies @@ -104,7 +104,7 @@ You can monitor a dynamic list of metrics by using the asterisk (\*) wildcard fo {{< figure src="/static/img/docs/v65/cloudwatch-dimension-wildcard.png" max-width="800px" class="docs-image--right" caption="CloudWatch dimension wildcard" >}} -In this example, the query returns all metrics in the namespace `AWS/EC2` with a metric name of `CPUUtilization` and ANY value for the `InstanceId` dimension are queried. This can help you monitor metrics for AWS resources, like EC2 instances or containers. For example, when new instances are created as part of an auto scaling event, they will automatically appear in the graph without needing to track the new instance IDs. This capability is currently limited to retrieving up to 100 metrics. +In this example, the query returns all metrics in the namespace `AWS/EC2` with a metric name of `CPUUtilization` and ANY value for the `InstanceId` dimension are queried. This can help you monitor metrics for AWS resources, like EC2 instances or containers. For example, when new instances are created as part of an auto scaling event, they will automatically appear in the graph without needing to track the new instance IDs. This capability is currently limited to retrieving up to 100 metrics. Click on `Show Query Preview` to see the search expression that is automatically built to support wildcards. To learn more about search expressions, visit the [CloudWatch documentation](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/search-expression-syntax.html). By default, the search expression is defined in such a way that the queried metrics must match the defined dimension names exactly. This means that in the example only metrics with exactly one dimension with name ‘InstanceId’ will be returned. diff --git a/package.json b/package.json index 595064cde2c..7ab5ad51e91 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,8 @@ "e2e:dev": "./e2e/start-and-run-suite dev", "test": "jest --notify --watch", "test:accessibility": "pa11y-ci --config .pa11yci.conf.js", + "test:accessibility-pr": "pa11y-ci --config .pa11yci-pr.conf.js", + "test:accessibility-report": "./scripts/generate-a11y-report.sh", "lint": "yarn run lint:ts && yarn run lint:sass", "lint:ts": "eslint . --ext .js,.tsx,.ts --cache", "lint:sass": "yarn run sass-lint -c public/sass/.sass-lint.yml 'public/sass/**/*.scss, packages/**/*.scss' -v -i '**/node_modules/**/*.scss'", @@ -171,6 +173,7 @@ "fs-extra": "9.1.0", "gaze": "1.1.3", "glob": "7.1.6", + "http-server": "13.0.1", "html-loader": "2.1.2", "html-webpack-harddisk-plugin": "2.0.0", "html-webpack-plugin": "5.3.2", diff --git a/scripts/drone/pipelines/pr.star b/scripts/drone/pipelines/pr.star index a69c730be25..9f741103006 100644 --- a/scripts/drone/pipelines/pr.star +++ b/scripts/drone/pipelines/pr.star @@ -23,6 +23,7 @@ load( 'memcached_integration_tests_step', 'benchmark_ldap_step', 'validate_scuemata_step', + 'test_a11y_frontend_step_pr', ) load( @@ -72,6 +73,7 @@ def pr_pipelines(edition): e2e_tests_server_step(edition=edition), e2e_tests_step(edition=edition), build_storybook_step(edition=edition, ver_mode=ver_mode), + test_a11y_frontend_step_pr(edition=edition), build_frontend_docs_step(edition=edition), build_docs_website_step(), copy_packages_for_docker_step(), diff --git a/scripts/drone/steps/lib.star b/scripts/drone/steps/lib.star index b6528910f03..f3827edb08f 100644 --- a/scripts/drone/steps/lib.star +++ b/scripts/drone/steps/lib.star @@ -423,6 +423,24 @@ def test_a11y_frontend_step(edition, port=3001): ], } +def test_a11y_frontend_step_pr(edition, port=3001): + return { + 'name': 'test-a11y-frontend-pr' + enterprise2_suffix(edition), + 'image': 'buildkite/puppeteer', + 'depends_on': [ + 'end-to-end-tests-server' + enterprise2_suffix(edition), + ], + 'environment': { + 'GRAFANA_MISC_STATS_API_KEY': from_secret('grafana_misc_stats_api_key'), + 'HOST': 'end-to-end-tests-server' + enterprise2_suffix(edition), + 'PORT': port, + }, + 'commands': [ + 'yarn wait-on http://$HOST:$PORT', + 'yarn -s test:accessibility-pr', + ], + } + def frontend_metrics_step(edition): if edition in ('enterprise', 'enterprise2'): return None diff --git a/scripts/generate-a11y-report.sh b/scripts/generate-a11y-report.sh new file mode 100755 index 00000000000..6131f639fc3 --- /dev/null +++ b/scripts/generate-a11y-report.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -x + +# Clean up old report +jsonReport="pa11y-ci-results.json" +if [ -f "$jsonReport" ] ; then + rm "$jsonReport" +fi + +# Clean up old folder +report="pa11y-ci-report/" + +if [ -d "$report" ] ; then + rm -R "$report" +fi + +# Run accessibility command +yarn wait-on http://localhost:3000 + +yarn run -s test:accessibility --json > pa11y-ci-results.json + +# Generate HTML report +pa11y-ci-reporter-html + +# Start local server +yarn http-server pa11y-ci-report -p 1234 diff --git a/yarn.lock b/yarn.lock index fde99727a8f..c9b228ae184 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7306,6 +7306,11 @@ base@^0.11.1: mixin-deep "^1.2.0" pascalcase "^0.1.1" +basic-auth@^1.0.3: + version "1.1.0" + resolved "https://registry.yarnpkg.com/basic-auth/-/basic-auth-1.1.0.tgz#45221ee429f7ee1e5035be3f51533f1cdfd29884" + integrity sha1-RSIe5Cn37h5QNb4/UVM/HN/SmIQ= + batch-processor@1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/batch-processor/-/batch-processor-1.0.0.tgz#75c95c32b748e0850d10c2b168f6bdbe9891ace8" @@ -8407,7 +8412,7 @@ colorette@^1.1.0, colorette@^1.2.1, colorette@^1.2.2, colorette@^1.3.0: resolved "https://registry.yarnpkg.com/colorette/-/colorette-1.3.0.tgz#ff45d2f0edb244069d3b772adeb04fed38d0a0af" integrity sha512-ecORCqbSFP7Wm8Y6lyqMJjexBQqXSF7SSeaTyGGphogUjBlFP9m9o08wy86HL2uB7fMTxtOUzLMk7ogKcxMg1w== -colors@^1.1.2: +colors@^1.1.2, colors@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/colors/-/colors-1.4.0.tgz#c50491479d4c1bdaed2c9ced32cf7c7dc2360f78" integrity sha512-a+UqTh4kgZg/SlGvfbzDHpgRu7AAQOmmqRHJnxhRZICKFUT91brVhNNt58CMWU9PsBbv3PDCZUHbVxuDiH2mtA== @@ -8859,6 +8864,11 @@ core-util-is@1.0.2, core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7" integrity sha1-tf1UIgqivFq1eqtxQMlAdUUDwac= +corser@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/corser/-/corser-2.0.1.tgz#8eda252ecaab5840dcd975ceb90d9370c819ff87" + integrity sha1-jtolLsqrWEDc2XXOuQ2TcMgZ/4c= + cosmiconfig@^5.0.0: version "5.2.1" resolved "https://registry.yarnpkg.com/cosmiconfig/-/cosmiconfig-5.2.1.tgz#040f726809c591e77a17c0a3626ca45b4f168b1a" @@ -12919,7 +12929,7 @@ hastscript@^6.0.0: property-information "^5.0.0" space-separated-tokens "^1.0.0" -he@1.2.0, he@1.2.x, he@^1.2.0: +he@1.2.0, he@1.2.x, he@^1.1.0, he@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== @@ -13226,7 +13236,7 @@ http-proxy-middleware@^2.0.0: is-plain-obj "^3.0.0" micromatch "^4.0.2" -http-proxy@^1.18.1: +http-proxy@^1.18.0, http-proxy@^1.18.1: version "1.18.1" resolved "https://registry.yarnpkg.com/http-proxy/-/http-proxy-1.18.1.tgz#401541f0534884bbf95260334e72f88ee3976549" integrity sha512-7mz/721AbnJwIVbnaSv1Cz3Am0ZLT/UBwkC92VlxhXv/k/BBQfM2fXElQNC27BVGr0uwUpplYPQM9LnaBMR5NQ== @@ -13235,6 +13245,24 @@ http-proxy@^1.18.1: follow-redirects "^1.0.0" requires-port "^1.0.0" +http-server@13.0.1: + version "13.0.1" + resolved "https://registry.yarnpkg.com/http-server/-/http-server-13.0.1.tgz#e7340d082925c4b1d6484c905d0df29d7d8ec16f" + integrity sha512-ke9rphoNuqsOCHy4tA3b3W4Yuxy7VUIXcTHSLz6bkMDAJPQD4twjEatquelJBIPwNhZuC3+FYj/+dSaGHdKTCw== + dependencies: + basic-auth "^1.0.3" + colors "^1.4.0" + corser "^2.0.1" + he "^1.1.0" + http-proxy "^1.18.0" + mime "^1.6.0" + minimist "^1.2.5" + opener "^1.5.1" + portfinder "^1.0.25" + secure-compare "3.0.1" + union "~0.5.0" + url-join "^2.0.5" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -16233,7 +16261,7 @@ mime-types@^2.1.12, mime-types@^2.1.27, mime-types@^2.1.30, mime-types@^2.1.31, dependencies: mime-db "1.49.0" -mime@1.6.0, mime@^1.4.1: +mime@1.6.0, mime@^1.4.1, mime@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1" integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg== @@ -17383,7 +17411,7 @@ opencollective-postinstall@^2.0.2: resolved "https://registry.yarnpkg.com/opencollective-postinstall/-/opencollective-postinstall-2.0.2.tgz#5657f1bede69b6e33a45939b061eb53d3c6c3a89" integrity sha512-pVOEP16TrAO2/fjej1IdOyupJY8KDUM1CvsaScRbw6oddvpQoOfGk4ywha0HKKVAD6RkW4x6Q+tNBwhf3Bgpuw== -opener@^1.5.2: +opener@^1.5.1, opener@^1.5.2: version "1.5.2" resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598" integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A== @@ -18146,7 +18174,7 @@ polished@^4.0.5: dependencies: "@babel/runtime" "^7.14.0" -portfinder@^1.0.28: +portfinder@^1.0.25, portfinder@^1.0.28: version "1.0.28" resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.28.tgz#67c4622852bd5374dd1dd900f779f53462fac778" integrity sha512-Se+2isanIcEqf2XMHjyUKskczxbPH7dQnlMjXX6+dybayyHvAf/TCgyMRlzf/B6QDhAEFOGes0pzRo3by4AbMA== @@ -19578,7 +19606,7 @@ qs@6.7.0: resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc" integrity sha512-VCdBRNFTX1fyE7Nb6FYoURo/SPe62QCaAyzJvUjwRaIsc+NePBEniHlvxFmmX56+HZphIGtV0XeCirBtpDrTyQ== -qs@^6.10.0, qs@^6.9.4: +qs@^6.10.0, qs@^6.4.0, qs@^6.9.4: version "6.10.1" resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.1.tgz#4931482fa8d647a5aab799c5271d2133b981fb6a" integrity sha512-M528Hph6wsSVOBiYUnGf+K/7w0hNshs/duGsNXPUCLH5XAqjEtiPGwNONLV0tBH8NoGb0mvD5JubnUTrujKDTg== @@ -21511,6 +21539,11 @@ search-query-parser@1.5.4: resolved "https://registry.yarnpkg.com/search-query-parser/-/search-query-parser-1.5.4.tgz#b474185e02717bee95f408e1003d0c1c932fcf55" integrity sha512-Mw3BpGU9SCid5uaEUTAz5y2hu7LT0VZbwmLKOH7bgoH+vxnP52kvX0xRXVwl14iCeL668mDeL92RxVX2dVX6Pg== +secure-compare@3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/secure-compare/-/secure-compare-3.0.1.tgz#f1a0329b308b221fae37b9974f3d578d0ca999e3" + integrity sha1-8aAymzCLIh+uN7mXTz1XjQypmeM= + select-hose@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/select-hose/-/select-hose-2.0.0.tgz#625d8658f865af43ec962bfc376a37359a4994ca" @@ -23616,6 +23649,13 @@ union-value@^1.0.0: is-extendable "^0.1.1" set-value "^2.0.1" +union@~0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/union/-/union-0.5.0.tgz#b2c11be84f60538537b846edb9ba266ba0090075" + integrity sha512-N6uOhuW6zO95P3Mel2I2zMsbsanvvtgn6jVqJv4vbVcz/JN0OkL9suomjQGmWtxJQXOCqUJvquc1sMeNz/IwlA== + dependencies: + qs "^6.4.0" + uniq@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/uniq/-/uniq-1.0.1.tgz#b31c5ae8254844a3a8281541ce2b04b865a734ff" @@ -23817,6 +23857,11 @@ urix@^0.1.0: resolved "https://registry.yarnpkg.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha1-2pN/emLiH+wf0Y1Js1wpNQZ6bHI= +url-join@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/url-join/-/url-join-2.0.5.tgz#5af22f18c052a000a48d7b82c5e9c2e2feeda728" + integrity sha1-WvIvGMBSoACkjXuCxenC4v7tpyg= + url-loader@^2.0.1: version "2.2.0" resolved "https://registry.yarnpkg.com/url-loader/-/url-loader-2.2.0.tgz#af321aece1fd0d683adc8aaeb27829f29c75b46e"