IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

Enable Cypress and Playwright retries (#29056)

Browse Source 
* Enable Cypress and Playwright retries
* Improve E2E commit status messages
* Demote unstable testcases, increase task timeout
This commit is contained in:
Mario Vitale
2024-11-05 11:46:06 +01:00
committed by GitHub
parent 4b9eb73bd2
commit 2975e529d5
12 changed files with 82 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
.github/workflows/e2e-tests-ci-template.yml vendored
View File

@@ -453,7 +453,11 @@ jobs:
fi
export TOTAL_SPECS=$(( PASSED + FAILED ))
export PASS_RATE=$(jq -r '100 * (env.PASSED | tonumber) / (env.TOTAL_SPECS | tonumber)' <<<'{}' | xargs -l printf '%.2f')
export COMMIT_STATUS_MESSAGE="Completed with pass rate: ${PASS_RATE}% (plus ${FAILED_EXPECTED} unrelated failures)"
if [ "$FAILED" = "0" ]; then
export COMMIT_STATUS_MESSAGE="All test cases passed"
else
export COMMIT_STATUS_MESSAGE="${FAILED} test cases failed. Please check the workflow logs"
fi
echo "passed=${PASSED:?}" >> $GITHUB_OUTPUT
echo "failed=${FAILED:?}" >> $GITHUB_OUTPUT
echo "failed_expected=${FAILED_EXPECTED:?}" >> $GITHUB_OUTPUT

2
e2e-tests/.ci/server.run_playwright.sh
View File

@@ -43,7 +43,7 @@ ${MME2E_DC_SERVER} exec -i -u "$MME2E_UID" -- playwright bash -c "cd e2e-tests/p
# Documentation on the results.json file: https://playwright.dev/docs/api/class-testcase#test-case-expected-status
# NB: the following line is needed only for compatibility reasons, to support RollingRelease tests for versions prior to v10.1.0
# It can be removed after releases <=v10.0.x are phased out
mv -v ../playwright/playwright-report/results.json ../playwright/results/results.json || true
mv -v ../playwright/playwright-report/results.json ../playwright/results/results.json 2>/dev/null || true
jq -f /dev/stdin ../playwright/results/results.json > ../playwright/results/summary.json <<EOF
{
passed: .stats.expected,

3
e2e-tests/cypress/cypress.config.ts
View File

@@ -9,8 +9,9 @@ export default defineConfig({
downloadsFolder: 'tests/downloads',
fixturesFolder: 'tests/fixtures',
numTestsKeptInMemory: 0,
retries: 2,
screenshotsFolder: 'tests/screenshots',
taskTimeout: 20000,
taskTimeout: 60000,
video: true,
videoCompression: true,
videosFolder: 'tests/videos',

1
e2e-tests/cypress/tests/integration/channels/notifications/direct_messages_do_not_add_indicator_spec.js
View File

@@ -7,7 +7,6 @@
// - Use element ID when selecting an element. Create one if none.
// ***************************************************************
// Stage: @prod
// Group: @channels @notifications
describe('Notifications', () => {

1
e2e-tests/cypress/tests/integration/channels/performance/channel_switch_spec.js
View File

@@ -7,7 +7,6 @@
// - Use element ID when selecting an element. Create one if none.
// ***************************************************************
// Stage: @prod
// Group: @channels @channel
import {measurePerformance} from './utils.js';

1
e2e-tests/cypress/tests/integration/playbooks/channels/rhs/status_update_spec.js
View File

@@ -6,7 +6,6 @@
// - [*] indicates an assertion (e.g. * Check the title)
// ***************************************************************
// Stage: @prod
// Group: @playbooks
/* eslint-disable no-only-tests/no-only-tests */

1
e2e-tests/cypress/tests/integration/playbooks/channels/slash_command/commands_spec.js
View File

@@ -6,7 +6,6 @@
// - [*] indicates an assertion (e.g. * Check the title)
// ***************************************************************
// Stage: @prod
// Group: @playbooks
import {switchToChannel} from '../../../channels/mark_as_unread/helpers';

1
e2e-tests/cypress/tests/integration/playbooks/channels/slash_command/todo_spec.js
View File

@@ -6,7 +6,6 @@
// - [*] indicates an assertion (e.g. * Check the title)
// ***************************************************************
// Stage: @prod
// Group: @playbooks
describe('channels > slash command > todo', {testIsolation: true}, () => {

1
e2e-tests/playwright/playwright.config.ts
View File

@@ -12,6 +12,7 @@ export default defineConfig({
globalSetup: require.resolve('./global_setup'),
forbidOnly: testConfig.isCI,
outputDir: './results/tests',
retries: testConfig.isCI ? 2 : 0,
testDir: 'tests',
timeout: duration.one_min,
workers: testConfig.workers,

2
e2e-tests/playwright/tests/accessibility/channels/intro_channel.spec.ts
View File

@@ -3,7 +3,7 @@
import {expect, test} from '@e2e-support/test_fixture';
test('Base channel accessibility', async ({pw, pages, axe}) => {
test.fixme('Base channel accessibility', async ({pw, pages, axe}) => {
// # Create and sign in a new user
const {user} = await pw.initSetup();

142
e2e-tests/playwright/tests/functional/channels/gif_picker/gif_picker_search.spec.ts
View File

@@ -3,95 +3,95 @@
import {expect, test} from '@e2e-support/test_fixture';
test('MM-T5445 Should search, select and post correct Gif when Gif picker is opened from center textbox', async ({
pw,
pages,
}) => {
const {user} = await pw.initSetup();
test.fixme(
'MM-T5445 Should search, select and post correct Gif when Gif picker is opened from center textbox',
async ({pw, pages}) => {
const {user} = await pw.initSetup();
// # Log in as a user in new browser context
const {page} = await pw.testBrowser.login(user);
// # Log in as a user in new browser context
const {page} = await pw.testBrowser.login(user);
// # Visit default channel page
const channelPage = new pages.ChannelsPage(page);
await channelPage.goto();
await channelPage.toBeVisible();
// # Visit default channel page
const channelPage = new pages.ChannelsPage(page);
await channelPage.goto();
await channelPage.toBeVisible();
// # Open emoji/gif picker
await channelPage.centerView.postCreate.openEmojiPicker();
await channelPage.emojiGifPickerPopup.toBeVisible();
// # Open emoji/gif picker
await channelPage.centerView.postCreate.openEmojiPicker();
await channelPage.emojiGifPickerPopup.toBeVisible();
// # Open gif tab
await channelPage.emojiGifPickerPopup.openGifTab();
// # Open gif tab
await channelPage.emojiGifPickerPopup.openGifTab();
// # Search for gif
await channelPage.emojiGifPickerPopup.searchGif('hello');
// # Search for gif
await channelPage.emojiGifPickerPopup.searchGif('hello');
// # Select the first gif
const {img: firstSearchGifResult, alt: altOfFirstSearchGifResult} =
await channelPage.emojiGifPickerPopup.getNthGif(0);
await firstSearchGifResult.click();
// # Select the first gif
const {img: firstSearchGifResult, alt: altOfFirstSearchGifResult} =
await channelPage.emojiGifPickerPopup.getNthGif(0);
await firstSearchGifResult.click();
// # Send the selected gif as a message
await channelPage.centerView.postCreate.sendMessage();
// # Send the selected gif as a message
await channelPage.centerView.postCreate.sendMessage();
// * Verify that last message has the gif
const lastPost = await channelPage.centerView.getLastPost();
await lastPost.toBeVisible();
await expect(lastPost.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
});
// * Verify that last message has the gif
const lastPost = await channelPage.centerView.getLastPost();
await lastPost.toBeVisible();
await expect(lastPost.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
},
);
test('MM-T5446 Should search, select and post correct Gif when Gif picker is opened from RHS textbox', async ({
pw,
pages,
}) => {
const {user} = await pw.initSetup();
test.fixme(
'MM-T5446 Should search, select and post correct Gif when Gif picker is opened from RHS textbox',
async ({pw, pages}) => {
const {user} = await pw.initSetup();
// # Log in as a user in new browser context
const {page} = await pw.testBrowser.login(user);
// # Log in as a user in new browser context
const {page} = await pw.testBrowser.login(user);
// # Visit default channel page
const channelPage = new pages.ChannelsPage(page);
await channelPage.goto();
await channelPage.toBeVisible();
// # Visit default channel page
const channelPage = new pages.ChannelsPage(page);
await channelPage.goto();
await channelPage.toBeVisible();
// # Send a message
await channelPage.centerView.postCreate.postMessage('Message to open RHS');
// # Send a message
await channelPage.centerView.postCreate.postMessage('Message to open RHS');
// # Open the last post sent in RHS
const lastPost = await channelPage.centerView.getLastPost();
await lastPost.hover();
await lastPost.postMenu.toBeVisible();
await lastPost.postMenu.reply();
// # Open the last post sent in RHS
const lastPost = await channelPage.centerView.getLastPost();
await lastPost.hover();
await lastPost.postMenu.toBeVisible();
await lastPost.postMenu.reply();
const sidebarRight = channelPage.sidebarRight;
await sidebarRight.toBeVisible();
const sidebarRight = channelPage.sidebarRight;
await sidebarRight.toBeVisible();
// # Send a message in the thread
await sidebarRight.postCreate.toBeVisible();
await sidebarRight.postCreate.writeMessage('Replying to a thread');
await sidebarRight.postCreate.sendMessage();
// # Send a message in the thread
await sidebarRight.postCreate.toBeVisible();
await sidebarRight.postCreate.writeMessage('Replying to a thread');
await sidebarRight.postCreate.sendMessage();
// # Open emoji/gif picker
await sidebarRight.postCreate.openEmojiPicker();
await channelPage.emojiGifPickerPopup.toBeVisible();
// # Open emoji/gif picker
await sidebarRight.postCreate.openEmojiPicker();
await channelPage.emojiGifPickerPopup.toBeVisible();
// # Open gif tab
await channelPage.emojiGifPickerPopup.openGifTab();
// # Open gif tab
await channelPage.emojiGifPickerPopup.openGifTab();
// # Search for gif
await channelPage.emojiGifPickerPopup.searchGif('hello');
// # Search for gif
await channelPage.emojiGifPickerPopup.searchGif('hello');
// # Select the first gif
const {img: firstSearchGifResult, alt: altOfFirstSearchGifResult} =
await channelPage.emojiGifPickerPopup.getNthGif(0);
await firstSearchGifResult.click();
// # Select the first gif
const {img: firstSearchGifResult, alt: altOfFirstSearchGifResult} =
await channelPage.emojiGifPickerPopup.getNthGif(0);
await firstSearchGifResult.click();
// # Send the selected gif as a message in the thread
await sidebarRight.postCreate.sendMessage();
// # Send the selected gif as a message in the thread
await sidebarRight.postCreate.sendMessage();
// * Verify that last message has the gif
const lastPostInRHS = await sidebarRight.getLastPost();
await lastPostInRHS.toBeVisible();
await expect(lastPostInRHS.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
});
// * Verify that last message has the gif
const lastPostInRHS = await sidebarRight.getLastPost();
await lastPostInRHS.toBeVisible();
await expect(lastPostInRHS.body.getByLabel('file thumbnail')).toHaveAttribute('alt', altOfFirstSearchGifResult);
},
);

2
e2e-tests/playwright/tests/visual/channels/intro_channel.spec.ts
View File

@@ -3,7 +3,7 @@
import {expect, test} from '@e2e-support/test_fixture';
test('Intro to channel as regular user', async ({pw, pages, browserName, viewport}, testInfo) => {
test.fixme('Intro to channel as regular user', async ({pw, pages, browserName, viewport}, testInfo) => {
// Create and sign in a new user
const {user} = await pw.initSetup();