mirror of
https://github.com/grafana/grafana.git
synced 2024-11-24 09:50:29 -06:00
E2E: Add plugin-e2e scenario verification tests (#79969)
* add playwright test and plugin-e2e * run tests in ci * add ds config tests * add panel edit tests * add annotation test * add variable edit page tests * add explore page tests * add panel plugin tests * add readme * remove comments * fix broken test * remove user.json * remove newline in starlark * fix lint issue * ignore failure of playwright tests * update code owners * add detailed error messages in every expect * update message frame * fix link * upload report to gcp * echo url * add playwright developer guide * bump plugin-e2e * add custom provisioning dir * update plugin-e2e * remove not used imports * fix typo * minor fixes * use latest version of plugin-e2e * fix broken link * use latest plugin-e2e * add feature toggle scenario verification tests * bump version * use auth file from package * fix type error * add panel data assertions * rename parent dir and bump version * fix codeowners * reset files * remove not used file * update plugin-e2e * separate tests per role * pass prov dir * skip using provisioning fixture * wip * fix permission test * move to e2e dir * fix path to readme * post comment with report url * format starlark * post comment with report url * post comment with report url * fix token * make test fail * fix exit code * bump version * bump to latest plugin-e2e * revert reporting message * remove comments * readding report comment * change exit code * format starlark * force test to fail * add new step that posts comment * fix link * use latest playwright image * fix failing test * format starlark * remove unused fixture Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com> --------- Co-authored-by: Marcus Andersson <marcus.andersson@grafana.com>
This commit is contained in:
Erik Sundell
GitHub
parent
b25667223c
commit
3e456127cb
112
.drone.yml
112
.drone.yml
@ -653,6 +653,60 @@ steps:
|
||||
- e2e/cloud-plugins-suite/azure-monitor.spec.ts
|
||||
repo:
|
||||
- grafana/grafana
|
||||
- commands:
|
||||
- sleep 10s
|
||||
- yarn e2e:playwright
|
||||
depends_on:
|
||||
- grafana-server
|
||||
environment:
|
||||
HOST: grafana-server
|
||||
PORT: "3001"
|
||||
PROV_DIR: /grafana/scripts/grafana-server/tmp/conf/provisioning
|
||||
failure: ignore
|
||||
image: mcr.microsoft.com/playwright:v1.41.2-jammy
|
||||
name: playwright-plugin-e2e
|
||||
- commands:
|
||||
- apt-get update
|
||||
- apt-get install -yq zip
|
||||
- printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json
|
||||
- gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json
|
||||
- gsutil cp -r ./playwright-report/. gs://releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report
|
||||
- export E2E_PLAYWRIGHT_REPORT_URL=https://storage.googleapis.com/releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report/index.html
|
||||
- "echo \"E2E Playwright report uploaded to: \n $${E2E_PLAYWRIGHT_REPORT_URL}\""
|
||||
depends_on:
|
||||
- playwright-plugin-e2e
|
||||
environment:
|
||||
GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY:
|
||||
from_secret: gcp_upload_artifacts_key
|
||||
failure: ignore
|
||||
image: google/cloud-sdk:431.0.0
|
||||
name: playwright-e2e-report-upload
|
||||
when:
|
||||
status:
|
||||
- success
|
||||
- failure
|
||||
- commands:
|
||||
- if [ ! -d ./playwright-report/trace ]; then echo 'all tests passed'; exit 0; fi
|
||||
- export E2E_PLAYWRIGHT_REPORT_URL=https://storage.googleapis.com/releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report/index.html
|
||||
- 'curl -L -X POST https://api.github.com/repos/grafana/grafana/issues/${DRONE_PULL_REQUEST}/comments
|
||||
-H "Accept: application/vnd.github+json" -H "Authorization: Bearer $${GITHUB_TOKEN}"
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" -d "{\"body\":\"❌ Failed to run Playwright
|
||||
plugin e2e tests. <br /> <br /> Click [here]($${E2E_PLAYWRIGHT_REPORT_URL}) to
|
||||
browse the Playwright report and trace viewer. <br /> For information on how to
|
||||
run Playwright tests locally, refer to the [Developer guide](https://github.com/grafana/grafana/blob/main/contribute/developer-guide.md#to-run-the-playwright-tests).
|
||||
\"}"'
|
||||
depends_on:
|
||||
- playwright-e2e-report-upload
|
||||
environment:
|
||||
GITHUB_TOKEN:
|
||||
from_secret: github_token
|
||||
failure: ignore
|
||||
image: byrnedo/alpine-curl:0.1.8
|
||||
name: playwright-e2e-report-post-link
|
||||
when:
|
||||
status:
|
||||
- success
|
||||
- failure
|
||||
- commands:
|
||||
- if [ -z `find ./e2e -type f -name *spec.ts.mp4` ]; then echo 'missing videos';
|
||||
false; fi
|
||||
@ -1929,6 +1983,60 @@ steps:
|
||||
- e2e/cloud-plugins-suite/azure-monitor.spec.ts
|
||||
repo:
|
||||
- grafana/grafana
|
||||
- commands:
|
||||
- sleep 10s
|
||||
- yarn e2e:playwright
|
||||
depends_on:
|
||||
- grafana-server
|
||||
environment:
|
||||
HOST: grafana-server
|
||||
PORT: "3001"
|
||||
PROV_DIR: /grafana/scripts/grafana-server/tmp/conf/provisioning
|
||||
failure: ignore
|
||||
image: mcr.microsoft.com/playwright:v1.41.2-jammy
|
||||
name: playwright-plugin-e2e
|
||||
- commands:
|
||||
- apt-get update
|
||||
- apt-get install -yq zip
|
||||
- printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json
|
||||
- gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json
|
||||
- gsutil cp -r ./playwright-report/. gs://releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report
|
||||
- export E2E_PLAYWRIGHT_REPORT_URL=https://storage.googleapis.com/releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report/index.html
|
||||
- "echo \"E2E Playwright report uploaded to: \n $${E2E_PLAYWRIGHT_REPORT_URL}\""
|
||||
depends_on:
|
||||
- playwright-plugin-e2e
|
||||
environment:
|
||||
GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY:
|
||||
from_secret: gcp_upload_artifacts_key
|
||||
failure: ignore
|
||||
image: google/cloud-sdk:431.0.0
|
||||
name: playwright-e2e-report-upload
|
||||
when:
|
||||
status:
|
||||
- success
|
||||
- failure
|
||||
- commands:
|
||||
- if [ ! -d ./playwright-report/trace ]; then echo 'all tests passed'; exit 0; fi
|
||||
- export E2E_PLAYWRIGHT_REPORT_URL=https://storage.googleapis.com/releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report/index.html
|
||||
- 'curl -L -X POST https://api.github.com/repos/grafana/grafana/issues/${DRONE_PULL_REQUEST}/comments
|
||||
-H "Accept: application/vnd.github+json" -H "Authorization: Bearer $${GITHUB_TOKEN}"
|
||||
-H "X-GitHub-Api-Version: 2022-11-28" -d "{\"body\":\"❌ Failed to run Playwright
|
||||
plugin e2e tests. <br /> <br /> Click [here]($${E2E_PLAYWRIGHT_REPORT_URL}) to
|
||||
browse the Playwright report and trace viewer. <br /> For information on how to
|
||||
run Playwright tests locally, refer to the [Developer guide](https://github.com/grafana/grafana/blob/main/contribute/developer-guide.md#to-run-the-playwright-tests).
|
||||
\"}"'
|
||||
depends_on:
|
||||
- playwright-e2e-report-upload
|
||||
environment:
|
||||
GITHUB_TOKEN:
|
||||
from_secret: github_token
|
||||
failure: ignore
|
||||
image: byrnedo/alpine-curl:0.1.8
|
||||
name: playwright-e2e-report-post-link
|
||||
when:
|
||||
status:
|
||||
- success
|
||||
- failure
|
||||
- commands:
|
||||
- if [ -z `find ./e2e -type f -name *spec.ts.mp4` ]; then echo 'missing videos';
|
||||
false; fi
|
||||
@ -4539,6 +4647,7 @@ steps:
|
||||
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM cypress/included:13.1.0
|
||||
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM jwilder/dockerize:0.6.1
|
||||
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM koalaman/shellcheck:stable
|
||||
- trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mcr.microsoft.com/playwright:v1.41.2-jammy
|
||||
depends_on:
|
||||
- authenticate-gcr
|
||||
image: aquasec/trivy:0.21.0
|
||||
@ -4573,6 +4682,7 @@ steps:
|
||||
- trivy --exit-code 1 --severity HIGH,CRITICAL cypress/included:13.1.0
|
||||
- trivy --exit-code 1 --severity HIGH,CRITICAL jwilder/dockerize:0.6.1
|
||||
- trivy --exit-code 1 --severity HIGH,CRITICAL koalaman/shellcheck:stable
|
||||
- trivy --exit-code 1 --severity HIGH,CRITICAL mcr.microsoft.com/playwright:v1.41.2-jammy
|
||||
depends_on:
|
||||
- authenticate-gcr
|
||||
environment:
|
||||
@ -4804,6 +4914,6 @@ kind: secret
|
||||
name: gcr_credentials
|
||||
---
|
||||
kind: signature
|
||||
hmac: 043eca32327fd48d9b479c3a51549b30f3825553c55df6220581def9bd22f513
|
||||
hmac: bce7b2ac72349019ec0c2ffbdca13c0591110ee53a3bfc72261df0ed05ca025a
|
||||
|
||||
...
|
||||
|
||||
2
.github/CODEOWNERS
vendored
2
.github/CODEOWNERS
vendored
@ -306,6 +306,7 @@
|
||||
/public/app/core/internationalization/ @grafana/grafana-frontend-platform
|
||||
/e2e/ @grafana/grafana-frontend-platform
|
||||
/e2e/cloud-plugins-suite/ @grafana/partner-datasources
|
||||
/e2e/plugin-e2e/plugin-e2e-api-tests/ @grafana/plugins-platform-frontend
|
||||
/packages/ @grafana/grafana-frontend-platform @grafana/plugins-platform-frontend
|
||||
/packages/grafana-e2e-selectors/ @grafana/grafana-frontend-platform
|
||||
/packages/grafana-e2e/ @grafana/grafana-frontend-platform
|
||||
@ -365,6 +366,7 @@
|
||||
/.husky/pre-commit @grafana/frontend-ops
|
||||
/cypress.config.js @grafana/grafana-frontend-platform
|
||||
/.levignore.js @grafana/plugins-platform-frontend
|
||||
playwright.config.ts @grafana/plugins-platform-frontend
|
||||
|
||||
|
||||
# public folder
|
||||
|
||||
6
.gitignore
vendored
6
.gitignore
vendored
@ -171,6 +171,11 @@ compilation-stats.json
|
||||
/e2e/build_results.zip
|
||||
/e2e/extensions
|
||||
/e2e/extensions-suite
|
||||
/test-results/
|
||||
/playwright-report/
|
||||
/blob-report/
|
||||
/playwright/.cache/
|
||||
/playwright/.auth/
|
||||
|
||||
# grafana server
|
||||
/scripts/grafana-server/server.log
|
||||
@ -209,3 +214,4 @@ public/app/plugins/**/dist/
|
||||
# Ignore transpiled JavaScript resulting from the generate-transformations.ts script.
|
||||
/public/app/features/transformers/docs/*.js
|
||||
/scripts/docs/generate-transformations.js
|
||||
|
||||
|
||||
@ -171,9 +171,11 @@ make test-go-integration-postgres
|
||||
|
||||
### Run end-to-end tests
|
||||
|
||||
The end to end tests in Grafana use [Cypress](https://www.cypress.io/) to run automated scripts in a headless Chromium browser. Read more about our [e2e framework](/contribute/style-guides/e2e.md).
|
||||
The end to end tests in Grafana use [Cypress](https://www.cypress.io/) and [Playwright](https://playwright.dev/) to run automated scripts in a browser. Read more about our Cypress [e2e framework](/contribute/style-guides/e2e.md).
|
||||
|
||||
To run the tests:
|
||||
#### Running Cypress tests
|
||||
|
||||
To run all tests in a headless Chromium browser.
|
||||
|
||||
```
|
||||
yarn e2e
|
||||
@ -197,6 +199,34 @@ To choose a single test to follow in the browser as it runs, use `yarn e2e:dev`
|
||||
yarn e2e:dev
|
||||
```
|
||||
|
||||
#### To run the Playwright tests:
|
||||
|
||||
**Note:** If you're using VS Code as your development editor, it's recommended to install the [Playwright test extension](https://marketplace.visualstudio.com/items?itemName=ms-playwright.playwright). It allows you to run, debug and generate Playwright tests from within the editor. For more information about the extension and how to install it, refer to the [Playwright documentation](https://playwright.dev/docs/getting-started-vscode).
|
||||
|
||||
Each version of Playwright needs specific versions of browser binaries to operate. You will need to use the Playwright CLI to install these browsers.
|
||||
|
||||
```
|
||||
yarn playwright install chromium
|
||||
```
|
||||
|
||||
To run all tests in a headless Chromium browser and display results in the terminal.
|
||||
|
||||
```
|
||||
yarn e2e:playwright
|
||||
```
|
||||
|
||||
For a better developer experience, open the Playwright UI where you can easily walk through each step of the test and visually see what was happening before, during and after each step.
|
||||
|
||||
```
|
||||
yarn e2e:playwright:ui
|
||||
```
|
||||
|
||||
To open the HTML reporter for the last test run session.
|
||||
|
||||
```
|
||||
yarn e2e:playwright:report
|
||||
```
|
||||
|
||||
## Configure Grafana for development
|
||||
|
||||
The default configuration, `defaults.ini`, is located in the `conf` directory.
|
||||
|
||||
8
e2e/plugin-e2e/plugin-e2e-api-tests/README.md
Normal file
8
e2e/plugin-e2e/plugin-e2e-api-tests/README.md
Normal file
@ -0,0 +1,8 @@
|
||||
# @grafana/plugin-e2e API tests
|
||||
|
||||
The purpose of the E2E tests in this directory is not to test the plugins per se - it's to verify that the fixtures, models and expect matchers provided by the [`@grafana/plugin-e2e`](https://github.com/grafana/plugin-tools/tree/main/packages/plugin-e2e) package are compatible with the latest version of Grafana. If you find that any of these tests are failing, it's probably due to one of the following reasons:
|
||||
|
||||
- you have changed a value of a selector defined in @grafana/e2e-selector
|
||||
- you have made structural changes to the UI
|
||||
|
||||
For information on how to address this, follow the instructions in the [contributing guidelines](https://github.com/grafana/plugin-tools/blob/main/packages/plugin-e2e/CONTRIBUTING.md#how-to-fix-broken-test-scenarios-after-changes-in-grafana) for the @grafana/plugin-e2e package in the plugin-tools repository.
|
||||
@ -0,0 +1,15 @@
|
||||
import { expect, test } from '@grafana/plugin-e2e';
|
||||
|
||||
import { formatExpectError } from '../errors';
|
||||
import { successfulAnnotationQuery } from '../mocks/queries';
|
||||
|
||||
test('annotation query data with mocked response', async ({ annotationEditPage, page }) => {
|
||||
annotationEditPage.mockQueryDataResponse(successfulAnnotationQuery);
|
||||
await annotationEditPage.datasource.set('gdev-testdata');
|
||||
await page.getByLabel('Scenario').last().fill('CSV Content');
|
||||
await page.keyboard.press('Tab');
|
||||
await expect(
|
||||
annotationEditPage.runQuery(),
|
||||
formatExpectError('Expected annotation query to execute successfully')
|
||||
).toBeOK();
|
||||
});
|
||||
@ -0,0 +1,35 @@
|
||||
import { expect, test } from '@grafana/plugin-e2e';
|
||||
|
||||
import { formatExpectError } from '../errors';
|
||||
|
||||
test.describe('test createDataSourceConfigPage fixture, saveAndTest and toBeOK matcher', () => {
|
||||
test('invalid credentials should return an error', async ({ createDataSourceConfigPage, page }) => {
|
||||
const configPage = await createDataSourceConfigPage({ type: 'prometheus' });
|
||||
await page.getByPlaceholder('http://localhost:9090').fill('http://localhost:9090');
|
||||
await expect(
|
||||
configPage.saveAndTest(),
|
||||
formatExpectError('Expected save data source config to fail when Prometheus server is not running')
|
||||
).not.toBeOK();
|
||||
});
|
||||
|
||||
test('valid credentials should return a 200 status code', async ({ createDataSourceConfigPage, page }) => {
|
||||
const configPage = await createDataSourceConfigPage({ type: 'prometheus' });
|
||||
configPage.mockHealthCheckResponse({ status: 200 });
|
||||
await page.getByPlaceholder('http://localhost:9090').fill('http://localhost:9090');
|
||||
await expect(
|
||||
configPage.saveAndTest(),
|
||||
formatExpectError('Expected data source config to be successfully saved')
|
||||
).toBeOK();
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('test data source with frontend only health check', () => {
|
||||
test('valid credentials should display a success alert on the page', async ({ createDataSourceConfigPage }) => {
|
||||
const configPage = await createDataSourceConfigPage({ type: 'testdata' });
|
||||
await configPage.saveAndTest({ skipWaitForResponse: true });
|
||||
await expect(
|
||||
configPage,
|
||||
formatExpectError('Expected data source config to display success alert after save')
|
||||
).toHaveAlert('success', { hasNotText: 'Datasource updated' });
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,15 @@
|
||||
import { expect, test } from '@grafana/plugin-e2e';
|
||||
|
||||
import { formatExpectError } from '../errors';
|
||||
|
||||
test('query data response should be OK when query is valid', async ({ explorePage }) => {
|
||||
await explorePage.datasource.set('gdev-testdata');
|
||||
await expect(explorePage.runQuery(), formatExpectError('Expected Explore query to execute successfully')).toBeOK();
|
||||
});
|
||||
|
||||
test('query data response should not be OK when query is invalid', async ({ explorePage }) => {
|
||||
await explorePage.datasource.set('gdev-testdata');
|
||||
const queryEditorRow = await explorePage.getQueryEditorRow('A');
|
||||
await queryEditorRow.getByLabel('Labels').fill('invalid-label-format');
|
||||
await expect(explorePage.runQuery(), formatExpectError('Expected Explore query to fail')).not.toBeOK();
|
||||
});
|
||||
@ -0,0 +1,17 @@
|
||||
import { expect, test } from '@grafana/plugin-e2e';
|
||||
|
||||
const TRUTHY_CUSTOM_TOGGLE = 'custom_toggle1';
|
||||
const FALSY_CUSTOM_TOGGLE = 'custom_toggle2';
|
||||
|
||||
// override the feature toggles defined in playwright.config.ts only for tests in this file
|
||||
test.use({
|
||||
featureToggles: {
|
||||
[TRUTHY_CUSTOM_TOGGLE]: true,
|
||||
[FALSY_CUSTOM_TOGGLE]: false,
|
||||
},
|
||||
});
|
||||
|
||||
test('should set and check feature toggles correctly', async ({ isFeatureToggleEnabled }) => {
|
||||
expect(await isFeatureToggleEnabled(TRUTHY_CUSTOM_TOGGLE)).toBeTruthy();
|
||||
expect(await isFeatureToggleEnabled(FALSY_CUSTOM_TOGGLE)).toBeFalsy();
|
||||
});
|
||||
@ -0,0 +1,110 @@
|
||||
import { expect, test, PanelEditPage, DashboardPage } from '@grafana/plugin-e2e';
|
||||
|
||||
import { formatExpectError } from '../errors';
|
||||
import { successfulDataQuery } from '../mocks/queries';
|
||||
|
||||
const REACT_TABLE_DASHBOARD = { uid: 'U_bZIMRMk' };
|
||||
|
||||
test.describe('panel edit page', () => {
|
||||
test('table panel data assertions with provisioned dashboard', async ({
|
||||
page,
|
||||
selectors,
|
||||
grafanaVersion,
|
||||
request,
|
||||
}) => {
|
||||
const panelEditPage = new PanelEditPage(
|
||||
{ page, selectors, grafanaVersion, request },
|
||||
{ dashboard: REACT_TABLE_DASHBOARD, id: '4' }
|
||||
);
|
||||
await panelEditPage.goto();
|
||||
await expect(
|
||||
panelEditPage.panel.locator,
|
||||
formatExpectError('Could not locate panel in panel edit page')
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
panelEditPage.panel.fieldNames,
|
||||
formatExpectError('Could not locate header elements in table panel')
|
||||
).toContainText(['Field', 'Max', 'Mean', 'Last']);
|
||||
});
|
||||
|
||||
test('table panel data assertions', async ({ panelEditPage }) => {
|
||||
await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200);
|
||||
await panelEditPage.datasource.set('gdev-testdata');
|
||||
await panelEditPage.setVisualization('Table');
|
||||
await panelEditPage.refreshPanel();
|
||||
await expect(
|
||||
panelEditPage.panel.locator,
|
||||
formatExpectError('Could not locate panel in panel edit page')
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
panelEditPage.panel.fieldNames,
|
||||
formatExpectError('Could not locate header elements in table panel')
|
||||
).toContainText(['col1', 'col2']);
|
||||
await expect(panelEditPage.panel.data, formatExpectError('Could not locate headers in table panel')).toContainText([
|
||||
'val1',
|
||||
'val2',
|
||||
'val3',
|
||||
'val4',
|
||||
]);
|
||||
});
|
||||
|
||||
test('timeseries panel - table view assertions', async ({ panelEditPage }) => {
|
||||
await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200);
|
||||
await panelEditPage.datasource.set('gdev-testdata');
|
||||
await panelEditPage.setVisualization('Time series');
|
||||
await panelEditPage.refreshPanel();
|
||||
await panelEditPage.toggleTableView();
|
||||
await expect(
|
||||
panelEditPage.panel.locator,
|
||||
formatExpectError('Could not locate panel in panel edit page')
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
panelEditPage.panel.fieldNames,
|
||||
formatExpectError('Could not locate header elements in table panel')
|
||||
).toContainText(['col1', 'col2']);
|
||||
await expect(
|
||||
panelEditPage.panel.data,
|
||||
formatExpectError('Could not locate data elements in table panel')
|
||||
).toContainText(['val1', 'val2', 'val3', 'val4']);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('dashboard page', () => {
|
||||
test('getting panel by title', async ({ page, selectors, grafanaVersion, request }) => {
|
||||
const dashboardPage = new DashboardPage({ page, selectors, grafanaVersion, request }, REACT_TABLE_DASHBOARD);
|
||||
await dashboardPage.goto();
|
||||
const panel = await dashboardPage.getPanelByTitle('Colored background');
|
||||
await expect(panel.fieldNames).toContainText(['Field', 'Max', 'Mean', 'Last']);
|
||||
});
|
||||
|
||||
test('getting panel by id', async ({ page, selectors, grafanaVersion, request }) => {
|
||||
const dashboardPage = new DashboardPage({ page, selectors, grafanaVersion, request }, REACT_TABLE_DASHBOARD);
|
||||
await dashboardPage.goto();
|
||||
const panel = await dashboardPage.getPanelById('4');
|
||||
await expect(panel.fieldNames, formatExpectError('Could not locate header elements in table panel')).toContainText([
|
||||
'Field',
|
||||
'Max',
|
||||
'Mean',
|
||||
'Last',
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('explore page', () => {
|
||||
test('table panel', async ({ explorePage }) => {
|
||||
const url =
|
||||
'left=%7B"datasource":"grafana","queries":%5B%7B"queryType":"randomWalk","refId":"A","datasource":%7B"type":"datasource","uid":"grafana"%7D%7D%5D,"range":%7B"from":"1547161200000","to":"1576364400000"%7D%7D&orgId=1';
|
||||
await explorePage.goto({
|
||||
queryParams: new URLSearchParams(url),
|
||||
});
|
||||
await expect(
|
||||
explorePage.timeSeriesPanel.locator,
|
||||
formatExpectError('Could not locate time series panel in explore page')
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
explorePage.tablePanel.locator,
|
||||
formatExpectError('Could not locate table panel in explore page')
|
||||
).toBeVisible();
|
||||
await expect(explorePage.tablePanel.fieldNames).toContainText(['time', 'A-series']);
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,85 @@
|
||||
import { expect, test } from '@grafana/plugin-e2e';
|
||||
|
||||
import { formatExpectError } from '../errors';
|
||||
import { successfulDataQuery } from '../mocks/queries';
|
||||
import { scenarios } from '../mocks/resources';
|
||||
|
||||
const PANEL_TITLE = 'Table panel E2E test';
|
||||
const TABLE_VIZ_NAME = 'Table';
|
||||
const STANDARD_OTIONS_CATEGORY = 'Standard options';
|
||||
const DISPLAY_NAME_LABEL = 'Display name';
|
||||
|
||||
test.describe('query editor query data', () => {
|
||||
test('query data response should be OK when query is valid', async ({ panelEditPage }) => {
|
||||
await panelEditPage.datasource.set('gdev-testdata');
|
||||
await expect(
|
||||
panelEditPage.refreshPanel(),
|
||||
formatExpectError('Expected panel query to execute successfully')
|
||||
).toBeOK();
|
||||
});
|
||||
|
||||
test('query data response should not be OK and panel error should be displayed when query is invalid', async ({
|
||||
panelEditPage,
|
||||
}) => {
|
||||
await panelEditPage.datasource.set('gdev-testdata');
|
||||
const queryEditorRow = await panelEditPage.getQueryEditorRow('A');
|
||||
await queryEditorRow.getByLabel('Labels').fill('invalid-label-format');
|
||||
await expect(panelEditPage.refreshPanel(), formatExpectError('Expected panel query to fail')).not.toBeOK();
|
||||
await expect(
|
||||
panelEditPage.panel.getErrorIcon(),
|
||||
formatExpectError('Expected panel error to be displayed after query execution')
|
||||
).toBeVisible();
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('query editor with mocked responses', () => {
|
||||
test('and resource `scenarios` is mocked', async ({ panelEditPage, selectors }) => {
|
||||
await panelEditPage.mockResourceResponse('scenarios', scenarios);
|
||||
await panelEditPage.datasource.set('gdev-testdata');
|
||||
const queryEditorRow = await panelEditPage.getQueryEditorRow('A');
|
||||
await queryEditorRow.getByLabel('Scenario').last().click();
|
||||
await expect(
|
||||
panelEditPage.getByTestIdOrAriaLabel(selectors.components.Select.option),
|
||||
formatExpectError('Expected certain select options to be displayed after clicking on the select input')
|
||||
).toHaveText(scenarios.map((s) => s.name));
|
||||
});
|
||||
|
||||
test('mocked query data response', async ({ panelEditPage, selectors }) => {
|
||||
await panelEditPage.mockQueryDataResponse(successfulDataQuery, 200);
|
||||
await panelEditPage.datasource.set('gdev-testdata');
|
||||
await panelEditPage.setVisualization(TABLE_VIZ_NAME);
|
||||
await panelEditPage.refreshPanel();
|
||||
await expect(
|
||||
panelEditPage.panel.getErrorIcon(),
|
||||
formatExpectError('Did not expect panel error to be displayed after query execution')
|
||||
).not.toBeVisible();
|
||||
await expect(
|
||||
panelEditPage.getByTestIdOrAriaLabel(selectors.components.Panels.Visualization.Table.body),
|
||||
formatExpectError('Expected certain select options to be displayed after clicking on the select input')
|
||||
).toHaveText('val1val2val3val4');
|
||||
});
|
||||
});
|
||||
|
||||
test.describe('edit panel plugin settings', () => {
|
||||
test('change viz to table panel, set panel title and collapse section', async ({
|
||||
panelEditPage,
|
||||
selectors,
|
||||
page,
|
||||
}) => {
|
||||
await panelEditPage.setVisualization(TABLE_VIZ_NAME);
|
||||
await expect(
|
||||
panelEditPage.getByTestIdOrAriaLabel(selectors.components.PanelEditor.toggleVizPicker),
|
||||
formatExpectError('Expected panel visualization to be set to table')
|
||||
).toHaveText(TABLE_VIZ_NAME);
|
||||
await panelEditPage.setPanelTitle(PANEL_TITLE);
|
||||
await expect(
|
||||
panelEditPage.getByTestIdOrAriaLabel(selectors.components.Panels.Panel.title(PANEL_TITLE)),
|
||||
formatExpectError('Expected panel title to be updated')
|
||||
).toBeVisible();
|
||||
await panelEditPage.collapseSection(STANDARD_OTIONS_CATEGORY);
|
||||
await expect(
|
||||
page.getByText(DISPLAY_NAME_LABEL),
|
||||
formatExpectError('Expected section to be collapsed')
|
||||
).toBeVisible();
|
||||
});
|
||||
});
|
||||
@ -0,0 +1,16 @@
|
||||
import { expect, test } from '@grafana/plugin-e2e';
|
||||
|
||||
import { formatExpectError } from '../errors';
|
||||
import { prometheusLabels } from '../mocks/resources';
|
||||
|
||||
test('variable query with mocked response', async ({ variableEditPage, page }) => {
|
||||
variableEditPage.mockResourceResponse('api/v1/labels?*', prometheusLabels);
|
||||
await variableEditPage.datasource.set('gdev-prometheus');
|
||||
await variableEditPage.getByTestIdOrAriaLabel('Query type').fill('Label names');
|
||||
await page.keyboard.press('Tab');
|
||||
await variableEditPage.runQuery();
|
||||
await expect(
|
||||
variableEditPage,
|
||||
formatExpectError('Expected variable edit page to display certain label names after query execution')
|
||||
).toDisplayPreviews(prometheusLabels.data);
|
||||
});
|
||||
@ -0,0 +1,15 @@
|
||||
import { expect, test } from '@grafana/plugin-e2e';
|
||||
|
||||
test('should redirect to start page when permissions to navigate to page is missing', async ({ page }) => {
|
||||
await page.goto('/');
|
||||
const homePageTitle = await page.title();
|
||||
await page.goto('/datasources', { waitUntil: 'networkidle' });
|
||||
expect(await page.title()).toEqual(homePageTitle);
|
||||
});
|
||||
|
||||
test('current user should have viewer role', async ({ page, request }) => {
|
||||
await page.goto('/');
|
||||
const response = await request.get('/api/user/orgs');
|
||||
await expect(response).toBeOK();
|
||||
await expect(await response.json()).toContainEqual(expect.objectContaining({ role: 'Viewer' }));
|
||||
});
|
||||
4
e2e/plugin-e2e/plugin-e2e-api-tests/errors.ts
Normal file
4
e2e/plugin-e2e/plugin-e2e-api-tests/errors.ts
Normal file
@ -0,0 +1,4 @@
|
||||
export const formatExpectError = (message: string) => {
|
||||
return `Error while verifying @grafana/plugin-e2e scenarios: ${message}.
|
||||
See https://github.com/grafana/grafana/blob/main/plugin-e2e/plugin-e2e-api-tests/README.md for more information.`;
|
||||
};
|
||||
77
e2e/plugin-e2e/plugin-e2e-api-tests/mocks/queries.ts
Normal file
77
e2e/plugin-e2e/plugin-e2e-api-tests/mocks/queries.ts
Normal file
@ -0,0 +1,77 @@
|
||||
export const successfulDataQuery = {
|
||||
results: {
|
||||
A: {
|
||||
status: 200,
|
||||
frames: [
|
||||
{
|
||||
schema: {
|
||||
refId: 'A',
|
||||
fields: [
|
||||
{
|
||||
name: 'col1',
|
||||
type: 'string',
|
||||
typeInfo: {
|
||||
frame: 'string',
|
||||
nullable: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'col2',
|
||||
type: 'string',
|
||||
typeInfo: {
|
||||
frame: 'string',
|
||||
nullable: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
data: {
|
||||
values: [
|
||||
['val1', 'val3'],
|
||||
['val2', 'val4'],
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
export const successfulAnnotationQuery = {
|
||||
results: {
|
||||
Anno: {
|
||||
status: 200,
|
||||
frames: [
|
||||
{
|
||||
schema: {
|
||||
refId: 'Anno',
|
||||
fields: [
|
||||
{
|
||||
name: 'time',
|
||||
type: 'time',
|
||||
typeInfo: {
|
||||
frame: 'time.Time',
|
||||
nullable: true,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: 'col2',
|
||||
type: 'string',
|
||||
typeInfo: {
|
||||
frame: 'string',
|
||||
nullable: true,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
data: {
|
||||
values: [
|
||||
[1702973084093, 1702973084099],
|
||||
['val1', 'val2'],
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
};
|
||||
19
e2e/plugin-e2e/plugin-e2e-api-tests/mocks/resources.ts
Normal file
19
e2e/plugin-e2e/plugin-e2e-api-tests/mocks/resources.ts
Normal file
@ -0,0 +1,19 @@
|
||||
export const scenarios = [
|
||||
{
|
||||
description: '',
|
||||
id: 'annotations',
|
||||
name: 'Annotations',
|
||||
stringInput: '',
|
||||
},
|
||||
{
|
||||
description: '',
|
||||
id: 'arrow',
|
||||
name: 'Load Apache Arrow Data',
|
||||
stringInput: '',
|
||||
},
|
||||
];
|
||||
|
||||
export const prometheusLabels = {
|
||||
status: 'success',
|
||||
data: ['__name__', 'action', 'active', 'address'],
|
||||
};
|
||||
@ -17,6 +17,9 @@
|
||||
"e2e:enterprise": "./e2e/start-and-run-suite enterprise",
|
||||
"e2e:enterprise:dev": "./e2e/start-and-run-suite enterprise dev",
|
||||
"e2e:enterprise:debug": "./e2e/start-and-run-suite enterprise debug",
|
||||
"e2e:playwright": "yarn playwright test",
|
||||
"e2e:playwright:ui": "yarn playwright test --ui",
|
||||
"e2e:playwright:report": "yarn playwright show-report",
|
||||
"test": "jest --notify --watch",
|
||||
"test:coverage": "jest --coverage",
|
||||
"test:coverage:changes": "jest --coverage --changedSince=origin/main",
|
||||
@ -73,7 +76,9 @@
|
||||
"@emotion/eslint-plugin": "11.11.0",
|
||||
"@grafana/eslint-config": "7.0.0",
|
||||
"@grafana/eslint-plugin": "link:./packages/grafana-eslint-rules",
|
||||
"@grafana/plugin-e2e": "0.18.0",
|
||||
"@grafana/tsconfig": "^1.3.0-rc1",
|
||||
"@playwright/test": "^1.41.2",
|
||||
"@pmmmwh/react-refresh-webpack-plugin": "0.5.11",
|
||||
"@react-types/button": "3.9.2",
|
||||
"@react-types/menu": "3.9.7",
|
||||
|
||||
65
playwright.config.ts
Normal file
65
playwright.config.ts
Normal file
@ -0,0 +1,65 @@
|
||||
import { defineConfig, devices } from '@playwright/test';
|
||||
import path, { dirname } from 'path';
|
||||
|
||||
import { PluginOptions } from '@grafana/plugin-e2e';
|
||||
|
||||
const testDirRoot = 'e2e/plugin-e2e/plugin-e2e-api-tests/';
|
||||
|
||||
export default defineConfig<PluginOptions>({
|
||||
fullyParallel: true,
|
||||
/* Retry on CI only */
|
||||
retries: process.env.CI ? 2 : 0,
|
||||
/* Opt out of parallel tests on CI. */
|
||||
workers: process.env.CI ? 1 : undefined,
|
||||
reporter: 'html',
|
||||
use: {
|
||||
baseURL: `http://${process.env.HOST || 'localhost'}:${process.env.PORT || 3000}`,
|
||||
trace: 'on-first-retry',
|
||||
httpCredentials: {
|
||||
username: 'admin',
|
||||
password: 'admin',
|
||||
},
|
||||
provisioningRootDir: path.join(process.cwd(), process.env.PROV_DIR ?? 'conf/provisioning'),
|
||||
},
|
||||
projects: [
|
||||
// Login to Grafana with admin user and store the cookie on disk for use in other tests
|
||||
{
|
||||
name: 'authenticate',
|
||||
testDir: `${dirname(require.resolve('@grafana/plugin-e2e'))}/auth`,
|
||||
testMatch: [/.*\.js/],
|
||||
},
|
||||
// Login to Grafana with new user with viewer role and store the cookie on disk for use in other tests
|
||||
{
|
||||
name: 'createUserAndAuthenticate',
|
||||
testDir: `${dirname(require.resolve('@grafana/plugin-e2e'))}/auth`,
|
||||
testMatch: [/.*\.js/],
|
||||
use: {
|
||||
user: {
|
||||
user: 'viewer',
|
||||
password: 'password',
|
||||
role: 'Viewer',
|
||||
},
|
||||
},
|
||||
},
|
||||
// Run all tests in parallel using user with admin role
|
||||
{
|
||||
name: 'admin',
|
||||
testDir: path.join(testDirRoot, '/as-admin-user'),
|
||||
use: {
|
||||
...devices['Desktop Chrome'],
|
||||
storageState: 'playwright/.auth/admin.json',
|
||||
},
|
||||
dependencies: ['authenticate'],
|
||||
},
|
||||
// Run all tests in parallel using user with viewer role
|
||||
{
|
||||
name: 'viewer',
|
||||
testDir: path.join(testDirRoot, '/as-viewer-user'),
|
||||
use: {
|
||||
...devices['Desktop Chrome'],
|
||||
storageState: 'playwright/.auth/viewer.json',
|
||||
},
|
||||
dependencies: ['createUserAndAuthenticate'],
|
||||
},
|
||||
],
|
||||
});
|
||||
@ -13,6 +13,9 @@ load(
|
||||
"frontend_metrics_step",
|
||||
"grafana_server_step",
|
||||
"identify_runner_step",
|
||||
"playwright_e2e_report_post_link",
|
||||
"playwright_e2e_report_upload",
|
||||
"playwright_e2e_tests_step",
|
||||
"publish_images_step",
|
||||
"release_canary_npm_packages_step",
|
||||
"store_storybook_step",
|
||||
@ -100,6 +103,9 @@ def build_e2e(trigger, ver_mode):
|
||||
cloud = "azure",
|
||||
trigger = trigger_oss,
|
||||
),
|
||||
playwright_e2e_tests_step(),
|
||||
playwright_e2e_report_upload(),
|
||||
playwright_e2e_report_post_link(),
|
||||
e2e_tests_artifacts(),
|
||||
build_storybook_step(ver_mode = ver_mode),
|
||||
test_a11y_frontend_step(ver_mode = ver_mode),
|
||||
|
||||
@ -360,6 +360,65 @@ def e2e_tests_artifacts():
|
||||
],
|
||||
}
|
||||
|
||||
def playwright_e2e_report_upload():
|
||||
return {
|
||||
"name": "playwright-e2e-report-upload",
|
||||
"image": images["cloudsdk"],
|
||||
"depends_on": [
|
||||
"playwright-plugin-e2e",
|
||||
],
|
||||
"failure": "ignore",
|
||||
"when": {
|
||||
"status": [
|
||||
"success",
|
||||
"failure",
|
||||
],
|
||||
},
|
||||
"environment": {
|
||||
"GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY": from_secret(gcp_upload_artifacts_key),
|
||||
},
|
||||
"commands": [
|
||||
"apt-get update",
|
||||
"apt-get install -yq zip",
|
||||
"printenv GCP_GRAFANA_UPLOAD_ARTIFACTS_KEY > /tmp/gcpkey_upload_artifacts.json",
|
||||
"gcloud auth activate-service-account --key-file=/tmp/gcpkey_upload_artifacts.json",
|
||||
"gsutil cp -r ./playwright-report/. gs://releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report",
|
||||
"export E2E_PLAYWRIGHT_REPORT_URL=https://storage.googleapis.com/releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report/index.html",
|
||||
'echo "E2E Playwright report uploaded to: \n $${E2E_PLAYWRIGHT_REPORT_URL}"',
|
||||
],
|
||||
}
|
||||
|
||||
def playwright_e2e_report_post_link():
|
||||
return {
|
||||
"name": "playwright-e2e-report-post-link",
|
||||
"image": images["curl"],
|
||||
"depends_on": [
|
||||
"playwright-e2e-report-upload",
|
||||
],
|
||||
"failure": "ignore",
|
||||
"when": {
|
||||
"status": [
|
||||
"success",
|
||||
"failure",
|
||||
],
|
||||
},
|
||||
"environment": {
|
||||
"GITHUB_TOKEN": from_secret("github_token"),
|
||||
},
|
||||
"commands": [
|
||||
# if the trace doesn't folder exists, it means that there are no failed tests.
|
||||
"if [ ! -d ./playwright-report/trace ]; then echo 'all tests passed'; exit 0; fi",
|
||||
# if it exists, we will post a comment on the PR with the link to the report
|
||||
"export E2E_PLAYWRIGHT_REPORT_URL=https://storage.googleapis.com/releng-pipeline-artifacts-dev/${DRONE_BUILD_NUMBER}/playwright-report/index.html",
|
||||
"curl -L " +
|
||||
"-X POST https://api.github.com/repos/grafana/grafana/issues/${DRONE_PULL_REQUEST}/comments " +
|
||||
'-H "Accept: application/vnd.github+json" ' +
|
||||
'-H "Authorization: Bearer $${GITHUB_TOKEN}" ' +
|
||||
'-H "X-GitHub-Api-Version: 2022-11-28" -d ' +
|
||||
'"{\\"body\\":\\"❌ Failed to run Playwright plugin e2e tests. <br /> <br /> Click [here]($${E2E_PLAYWRIGHT_REPORT_URL}) to browse the Playwright report and trace viewer. <br /> For information on how to run Playwright tests locally, refer to the [Developer guide](https://github.com/grafana/grafana/blob/main/contribute/developer-guide.md#to-run-the-playwright-tests). \\"}"',
|
||||
],
|
||||
}
|
||||
|
||||
def upload_cdn_step(ver_mode, trigger = None):
|
||||
"""Uploads CDN assets using the Grafana build tool.
|
||||
|
||||
@ -786,6 +845,25 @@ def cloud_plugins_e2e_tests_step(suite, cloud, trigger = None):
|
||||
step = dict(step, when = when)
|
||||
return step
|
||||
|
||||
def playwright_e2e_tests_step():
|
||||
return {
|
||||
"environment": {
|
||||
"PORT": "3001",
|
||||
"HOST": "grafana-server",
|
||||
"PROV_DIR": "/grafana/scripts/grafana-server/tmp/conf/provisioning",
|
||||
},
|
||||
"name": "playwright-plugin-e2e",
|
||||
"image": images["playwright"],
|
||||
"failure": "ignore",
|
||||
"depends_on": [
|
||||
"grafana-server",
|
||||
],
|
||||
"commands": [
|
||||
"sleep 10s", # it seems sometimes that grafana-server is not actually ready when the step starts, so waiting for a few seconds before running the tests
|
||||
"yarn e2e:playwright",
|
||||
],
|
||||
}
|
||||
|
||||
def build_docs_website_step():
|
||||
return {
|
||||
"name": "build-docs-website",
|
||||
|
||||
@ -33,4 +33,5 @@ images = {
|
||||
"cypress": "cypress/included:13.1.0",
|
||||
"dockerize": "jwilder/dockerize:0.6.1",
|
||||
"shellcheck": "koalaman/shellcheck:stable",
|
||||
"playwright": "mcr.microsoft.com/playwright:v1.41.2-jammy",
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user