CI: Allow failed Playwright tests to fail entire build (#84244)

* bump plugin-e2e and fix failing test

* do not ignore failures

* generate trace on every test

* force test to fail for debugging purposes

* fix broken test

* regenerate drone file

.drone.yml

@@ -665,7 +665,6 @@ steps:
     HOST: grafana-server
     PORT: "3001"
     PROV_DIR: /grafana/scripts/grafana-server/tmp/conf/provisioning
-  failure: ignore
   image: mcr.microsoft.com/playwright:v1.41.2-jammy
   name: playwright-plugin-e2e
 - commands:
@@ -1996,7 +1995,6 @@ steps:
     HOST: grafana-server
     PORT: "3001"
     PROV_DIR: /grafana/scripts/grafana-server/tmp/conf/provisioning
-  failure: ignore
   image: mcr.microsoft.com/playwright:v1.41.2-jammy
   name: playwright-plugin-e2e
 - commands:
@@ -4924,6 +4922,6 @@ kind: secret
 name: gcr_credentials
 ---
 kind: signature
-hmac: 3fa4360bd3c21fbc95bd3c2b63130541f9ea7c0aa56eb95dfe49c4f579aac4bb
+hmac: 9f04ec823cb2fd9ead455b7a0ede80a360ae503f85438e6ca1b7b14621a221ad
 
 ...

e2e/plugin-e2e/plugin-e2e-api-tests/as-admin-user/datasourceConfigPage.spec.ts

@@ -24,9 +24,18 @@ test.describe('test createDataSourceConfigPage fixture, saveAndTest and toBeOK m
 });
 
 test.describe('test data source with frontend only health check', () => {
-  test('valid credentials should display a success alert on the page', async ({ createDataSourceConfigPage }) => {
+  test('valid credentials should display a success alert on the page', async ({
-    const configPage = await createDataSourceConfigPage({ type: 'testdata' });
+    createDataSourceConfigPage,
-    await configPage.saveAndTest({ skipWaitForResponse: true });
+    page,
+    selectors,
+  }) => {
+    const configPage = await createDataSourceConfigPage({ type: 'zipkin' });
+    const healthCheckPath = `${selectors.apis.DataSource.proxy(configPage.datasource.uid)}/api/v2/services`;
+    await page.route(healthCheckPath, async (route) => {
+      await route.fulfill({ status: 200, body: 'OK' });
+    });
+    await page.getByPlaceholder('http://localhost:9411').fill('http://localhost:9411');
+    await expect(configPage.saveAndTest({ path: healthCheckPath })).toBeOK();
     await expect(
       configPage,
       formatExpectError('Expected data source config to display success alert after save')

package.json

@@ -76,7 +76,7 @@
     "@emotion/eslint-plugin": "11.11.0",
     "@grafana/eslint-config": "7.0.0",
     "@grafana/eslint-plugin": "link:./packages/grafana-eslint-rules",
-    "@grafana/plugin-e2e": "0.18.0",
+    "@grafana/plugin-e2e": "^0.19.0",
     "@grafana/tsconfig": "^1.3.0-rc1",
     "@playwright/test": "^1.41.2",
     "@pmmmwh/react-refresh-webpack-plugin": "0.5.11",

playwright.config.ts

@@ -14,7 +14,7 @@ export default defineConfig<PluginOptions>({
   reporter: 'html',
   use: {
     baseURL: `http://${process.env.HOST || 'localhost'}:${process.env.PORT || 3000}`,
-    trace: 'on-first-retry',
+    trace: 'retain-on-failure',
     httpCredentials: {
       username: 'admin',
       password: 'admin',

scripts/drone/steps/lib.star

@@ -854,7 +854,6 @@ def playwright_e2e_tests_step():
         },
         "name": "playwright-plugin-e2e",
         "image": images["playwright"],
-        "failure": "ignore",
         "depends_on": [
             "grafana-server",
         ],

yarn.lock

@@ -3901,16 +3901,16 @@ __metadata:
   languageName: unknown
   linkType: soft
 
-"@grafana/plugin-e2e@npm:0.18.0":
+"@grafana/plugin-e2e@npm:^0.19.0":
-  version: 0.18.0
+  version: 0.19.0
-  resolution: "@grafana/plugin-e2e@npm:0.18.0"
+  resolution: "@grafana/plugin-e2e@npm:0.19.0"
   dependencies:
     semver: "npm:^7.5.4"
     uuid: "npm:^9.0.1"
     yaml: "npm:^2.3.4"
   peerDependencies:
     "@playwright/test": ^1.41.2
-  checksum: 10/bb34d74dd802ed7dd80db2de2b8051e8f961a9e0510b3555ee8be6e2c5cc46e3751de1235c4184f7ebbe00cbaa0c280e9013fab5ba167d3f8a8639191b5174ea
+  checksum: 10/d47913d4abd07b245892d834b06ca44ecec767151134575d9e2679b93fad190cd6957542f6f0ddd12eaa4199b3282b557ddbb0ba88df13eb107986e35ed17d62
   languageName: node
   linkType: hard
 
@@ -18251,7 +18251,7 @@ __metadata:
     "@grafana/lezer-logql": "npm:0.2.3"
     "@grafana/monaco-logql": "npm:^0.0.7"
     "@grafana/o11y-ds-frontend": "workspace:*"
-    "@grafana/plugin-e2e": "npm:0.18.0"
+    "@grafana/plugin-e2e": "npm:^0.19.0"
     "@grafana/prometheus": "workspace:*"
     "@grafana/runtime": "workspace:*"
     "@grafana/scenes": "npm:^3.11.0"