Chore: re-enable pa11y-ci for PRs (#41723)

* Chore: re-enable pa11y-ci for PRs

* Chore: update signature

* Chore: revert wait times

* Chore: use same hash as docker image

.drone.yml

@@ -228,7 +228,7 @@ steps:
       from_secret: grafana_misc_stats_api_key
     HOST: end-to-end-tests-server
     PORT: 3001
-  failure: ignore
+  failure: always
   image: hugohaggmark/docker-puppeteer
   name: test-a11y-frontend
 - commands:
@@ -3500,6 +3500,6 @@ kind: secret
 name: drone_token
 ---
 kind: signature
-hmac: 434491b80be6d7ca03ed7ca8b9978794ac1bc9ee5037d42bcce9b245c1f76579
+hmac: 9f5633098fb9d66f85dd2278290c5886e8863d348bc66f3ebf302c0edb7b5f15
 
 ...

.pa11yci-pr.conf.js

@@ -13,6 +13,7 @@ var config = {
       url: '${HOST}/login',
       wait: 500,
       rootElement: '.main-view',
+      threshold: 12,
     },
     {
       url: '${HOST}/login',
@@ -24,37 +25,37 @@ var config = {
         "click element button[aria-label='Login button']",
         "wait for element [aria-label='Skip change password button'] to be visible",
       ],
-      threshold: 1,
+      threshold: 14,
       rootElement: '.main-view',
     },
     {
       url: '${HOST}/?orgId=1',
       wait: 500,
-      threshold: 7,
+      threshold: 1,
     },
     {
       url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 2,
+      threshold: 51,
     },
     {
       url: '${HOST}/d/O6f11TZWk/panel-tests-bar-gauge?orgId=1&editview=settings',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 10,
+      threshold: 54,
     },
     {
       url: '${HOST}/?orgId=1&search=open',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 14,
+      threshold: 8,
     },
     {
       url: '${HOST}/alerting/list',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 6,
+      threshold: 1,
     },
     {
       url: '${HOST}/datasources',
@@ -66,13 +67,13 @@ var config = {
       url: '${HOST}/org/users',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 3,
+      threshold: 0,
     },
     {
       url: '${HOST}/org/teams',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 1,
+      threshold: 0,
     },
     {
       url: '${HOST}/plugins',
@@ -84,19 +85,19 @@ var config = {
       url: '${HOST}/org',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 2,
+      threshold: 3,
     },
     {
       url: '${HOST}/org/apikeys',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 3,
+      threshold: 1,
     },
     {
       url: '${HOST}/dashboards',
       wait: 500,
       rootElement: '.main-view',
-      threshold: 7,
+      threshold: 2,
     },
   ],
 };

scripts/drone/steps/lib.star

@@ -444,10 +444,12 @@ def test_a11y_frontend_step(ver_mode, edition, port=3001):
     commands = [
         'yarn wait-on http://$HOST:$PORT',
     ]
+    failure = 'ignore'
     if ver_mode == 'pr':
         commands.extend([
             'pa11y-ci --config .pa11yci-pr.conf.js',
         ])
+        failure = 'always'
     else:
         commands.extend([
             'pa11y-ci --config .pa11yci.conf.js --json > pa11y-ci-results.json',
@@ -464,7 +466,7 @@ def test_a11y_frontend_step(ver_mode, edition, port=3001):
             'HOST': 'end-to-end-tests-server' + enterprise2_suffix(edition),
             'PORT': port,
         },
-        'failure': 'ignore',
+        'failure': failure,
         'commands': commands,
     }
 

scripts/generate-a11y-report.sh

@@ -27,7 +27,7 @@ else
 fi
 
 # Run accessibility command
-yarn dlx --quiet pa11y-ci@pa11y/pa11y-ci#5c842cf1b9fe2867b70ff5354851d985be8d71c4 --config .pa11yci.conf.js --json > pa11y-ci-results.json
+yarn dlx --quiet pa11y-ci@pa11y/pa11y-ci#6b2d4f54efe445ad551472acc1877fe7542ac085 --config .pa11yci.conf.js --json > pa11y-ci-results.json
 
 # Generate HTML report
 yarn dlx pa11y-ci-reporter-html@3.0.1 pa11y-ci-reporter-html