A11y - Add Pa11y CI in main pipeline to collect a11y issues (#37741)

Co-authored-by: 
Hugo Häggmark <hugo.haggmark@gmail.com>
thisisobate <obasiuche62@gmail.com>

scripts/ci-frontend-metrics.sh

@@ -2,6 +2,7 @@
 set -e
 
 ERROR_COUNT="$(./node_modules/.bin/tsc --project tsconfig.json --noEmit --strict true | grep -oP 'Found \K(\d+)')"
+ACCESSIBILITY_ERRORS="$(grep -oP '\"errors\":(\d+),' pa11y-ci-results.json | grep -oP '\d+')"
 DIRECTIVES="$(grep -r -o  directive public/app/ | wc -l)"
 CONTROLLERS="$(grep -r -oP 'class .*Ctrl' public/app/ | wc -l)"
 STORIES_COUNT="$(find ./packages/grafana-ui/src/components -name "*.story.tsx" | wc -l)"
@@ -19,6 +20,7 @@ MED_VULNERABILITIES="$(echo "${VULNERABILITY_AUDIT}" | sed -n '2p')"
 HIGH_VULNERABILITIES="$(echo "${VULNERABILITY_AUDIT}" | sed -n '3p')"
 
 echo -e "Typescript errors: $ERROR_COUNT"
+echo -e "Accessibility errors: $ACCESSIBILITY_ERRORS"
 echo -e "Directives: $DIRECTIVES"
 echo -e "Controllers: $CONTROLLERS"
 echo -e "Stories: $STORIES_COUNT"

scripts/lib.star

@@ -481,6 +481,24 @@ def test_frontend_step():
         ],
     }
 
+def test_a11y_frontend_step(edition, port=3001):
+    return {
+        'name': 'test-a11y-frontend' + enterprise2_sfx(edition),
+        'image': 'buildkite/puppeteer',
+        'depends_on': [
+          'end-to-end-tests-server' + enterprise2_sfx(edition),
+        ],
+         'environment': {
+            'GRAFANA_MISC_STATS_API_KEY': from_secret('grafana_misc_stats_api_key'),
+            'HOST': 'end-to-end-tests-server' + enterprise2_sfx(edition),
+            'PORT': port,
+        },
+        'failure': 'ignore',
+        'commands': [
+            'yarn -s test:accessibility --json > pa11y-ci-results.json',
+        ],
+    }
+
 def frontend_metrics_step(edition):
     if edition in ('enterprise', 'enterprise2'):
         return None
@@ -489,7 +507,7 @@ def frontend_metrics_step(edition):
         'name': 'publish-frontend-metrics',
         'image': build_image,
         'depends_on': [
-            'build-frontend',
+            'test-a11y-frontend' + enterprise2_sfx(edition),
         ],
         'environment': {
             'GRAFANA_MISC_STATS_API_KEY': from_secret('grafana_misc_stats_api_key'),
@@ -500,7 +518,6 @@ def frontend_metrics_step(edition):
         ],
     }
 
-
 def codespell_step():
     return {
         'name': 'codespell',

scripts/main.star

@@ -34,7 +34,8 @@ load(
     'notify_pipeline',
     'integration_test_services',
     'upload_cdn',
-    'validate_scuemata'
+    'validate_scuemata',
+    'test_a11y_frontend_step'
 )
 
 ver_mode = 'main'
@@ -72,6 +73,7 @@ def get_steps(edition, is_downstream=False):
         e2e_tests_step(edition=edition),
         build_storybook_step(edition=edition, ver_mode=ver_mode),
         publish_storybook_step(edition=edition, ver_mode=ver_mode),
+        test_a11y_frontend_step(edition=edition),
         frontend_metrics_step(edition=edition),
         build_frontend_docs_step(edition=edition),
         copy_packages_for_docker_step(),