Remove deprecated Playwright/Cypress v1 dispatch (#37413)

* Remove deprecated Playwright/Cypress v1 e2e CI dispatch

Deletes the legacy in-job execution templates and their calculate-results
actions, and unconditionally routes both wrappers to the v2 (Test System IO)
templates. Test suites and local-dev tooling are untouched.

* remove v2 references

* remove aws dependencies in e2e tests, remove deprecated workflows
This commit is contained in:
sabril
2026-07-14 10:55:18 +08:00
committed by GitHub
parent 5634530e64
commit bd0d8866d6
45 changed files with 628 additions and 62867 deletions
-374
View File
@@ -1,374 +0,0 @@
---
name: E2E Tests
on:
# For PRs, this workflow gets triggered from the Argo Events platform.
# Check the following repo for details: https://github.com/mattermost/delivery-platform
workflow_dispatch:
inputs:
ref:
type: string
description: Git ref to test. Must be a full commit SHA for PR testing, and a tag for release testing. Ignored for daily tests.
required: false
PR_NUMBER:
type: string
description: PR number (if applicable)
required: false
ROLLING_RELEASE_FROM_TAG:
type: string
description: Mattermost release git tag for RollingRelease tests. Optional.
required: false
MM_ENV:
type: string
required: false
description: A comma-separated list of environment variables to set for the server. Spaces are not supported.
MM_SERVICE_OVERRIDES:
type: string
required: false
description: A comma-separated list of service overrides. E.g. "-elasticsearch,+opensearch"
REPORT_TYPE:
type: choice
description: The context this report is being generated in
options:
- PR
- RELEASE
- RELEASE_CLOUD
- MASTER
- MASTER_UNSTABLE
- CLOUD
- CLOUD_UNSTABLE
- NONE
default: NONE
RUN_CYPRESS:
type: string
description: Enable Cypress run
default: "true"
RUN_PLAYWRIGHT:
type: string
description: Enable Playwright run
default: "true"
FIPS_ENABLED:
type: string
description: When true, use mattermost-enterprise-fips-edition image for testing instead of standard enterprise edition
default: "false"
required: false
concurrency:
group: "${{ github.workflow }}-${{ inputs.REPORT_TYPE }}-${{ inputs.FIPS_ENABLED }}-${{ inputs.PR_NUMBER || inputs.ref }}-${{ inputs.MM_ENV }}"
cancel-in-progress: true
permissions: {}
jobs:
generate-test-variables:
runs-on: ubuntu-24.04
permissions:
contents: read
issues: write
pull-requests: write
defaults:
run:
shell: bash
outputs:
commit_sha: "${{ steps.generate.outputs.commit_sha }}"
BRANCH: "${{ steps.generate.outputs.BRANCH }}"
SERVER_IMAGE: "${{ steps.generate.outputs.SERVER_IMAGE }}"
status_check_context: "${{ steps.generate.outputs.status_check_context }}"
workers_number: "${{ steps.generate.outputs.workers_number }}"
server_uppercase: "${{ steps.generate.outputs.server_uppercase }}" # Required for license selection
SERVER: "${{ steps.generate.outputs.SERVER }}"
ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER_CYPRESS: "${{ steps.generate.outputs.TEST_FILTER_CYPRESS }}"
TEST_FILTER_PLAYWRIGHT: "tests"
BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}"
REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}"
TESTCASE_FAILURE_FATAL: "${{ steps.generate.outputs.TESTCASE_FAILURE_FATAL }}"
ROLLING_RELEASE_commit_sha: "${{ steps.generate.outputs.ROLLING_RELEASE_commit_sha }}"
ROLLING_RELEASE_SERVER_IMAGE: "${{ steps.generate.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
CYCLE_URL: "${{steps.generate.outputs.CYCLE_URL}}"
FIPS_SUFFIX: "${{ steps.generate.outputs.FIPS_SUFFIX }}"
env:
GH_TOKEN: "${{ github.token }}"
REF: "${{ inputs.ref || github.sha }}"
PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
ROLLING_RELEASE_FROM_TAG: "${{ inputs.ROLLING_RELEASE_FROM_TAG }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
FIPS_ENABLED: "${{ inputs.FIPS_ENABLED }}"
# We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
TEST_FILTER_CYPRESS_PR: >-
--stage="@prod"
--excludeGroup="@te_only,@cloud_only,@high_availability"
--sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap"
--sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
TEST_FILTER_CYPRESS_PROD_ONPREM: >-
--stage="@prod"
--excludeGroup="@te_only,@cloud_only,@high_availability"
--sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
--sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
TEST_FILTER_CYPRESS_PROD_CLOUD: >-
--stage="@prod"
--excludeGroup="@not_cloud,@cloud_trial,@e20_only,@te_only,@high_availability,@license_removal"
--sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
--sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa"
MM_ENV: "${{ inputs.MM_ENV || '' }}"
MM_SERVICE_OVERRIDES: "${{ inputs.MM_SERVICE_OVERRIDES }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: "${{ inputs.ref || github.sha }}"
fetch-depth: 0
- name: ci/generate-test-variables
id: generate
env:
GITHUB_RUN_ID: ${{ github.run_id }}
GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
GITHUB_SERVER_URL: ${{ github.server_url }}
GITHUB_REPOSITORY: ${{ github.repository }}
run: |
MM_ENV_HASH=$(md5sum -z <<<"$MM_ENV" | cut -c-8)
TESTCASE_FAILURE_FATAL="true"
if grep -q CLOUD <<<"$REPORT_TYPE"; then
SERVER=cloud
else
SERVER=onprem
fi
case "$REPORT_TYPE" in
NONE | PR)
### Populate support variables
_COMMIT_SHA_COMPUTED=$(git rev-parse --verify "$REF") # NB: not actually used for resolving the commit; it's only to double check the value of 'inputs.ref'
### For image tag generation: utilize 'inputs.ref', assume that it is a full commit SHA
COMMIT_SHA="${REF}"
BRANCH="server-pr-${PR_NUMBER}" # For reference, the real branch name may be retrievable with command: 'jq -r .head.ref <pr.json'
SERVER_IMAGE_TAG="${COMMIT_SHA::7}"
SERVER_IMAGE_ORG=mattermostdevelopment
BUILD_ID_SUFFIX="${REPORT_TYPE@L}-${SERVER}-ent"
WORKERS_NUMBER=20
TEST_FILTER_CYPRESS="$TEST_FILTER_CYPRESS_PR"
COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
### Run sanity assertions after variable generations
[ "$REF" = "${_COMMIT_SHA_COMPUTED}" ] # 'inputs.ref' must be a full commit hash, and the commit must exist
[ "$REPORT_TYPE" != "PR" ] || [ "$PR_NUMBER" -gt "0" ] # If report type is PR, then PR_NUMBER must be set to a number
;;
MASTER | MASTER_UNSTABLE | CLOUD | CLOUD_UNSTABLE)
### Populate support variables
_IS_TEST_UNSTABLE=$(sed -n -E 's/^.*(UNSTABLE).*$/\1/p' <<< "$REPORT_TYPE") # The variable's value is 'UNSTABLE' if report type is for unstable tests, otherwise it's empty
_TEST_FILTER_CYPRESS_VARIABLE="TEST_FILTER_CYPRESS_PROD_${SERVER@U}"
### For ref and image tag generation: ignore 'inputs.ref', and use master branch directly. Note that 'COMMIT_SHA' will be used for reporting the test result, and for checking out the testing scripts and test cases
COMMIT_SHA="$(git rev-parse --verify origin/master)"
BRANCH=master
SERVER_IMAGE_TAG=master
SERVER_IMAGE_ORG=mattermostdevelopment
BUILD_ID_SUFFIX="${_IS_TEST_UNSTABLE:+unstable-}daily-${SERVER}-ent"
BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
WORKERS_NUMBER=10 # Daily tests are not time critical, and it's more efficient to run on fewer workers
TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}"
TM4J_ENABLE=true
COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
[ -z "$_IS_TEST_UNSTABLE" ] || TESTCASE_FAILURE_FATAL="" # Assert that tests are stable. If they are not, the status check will be always green
;;
RELEASE | RELEASE_CLOUD)
### Populate support variables
_TEST_FILTER_CYPRESS_VARIABLE="TEST_FILTER_CYPRESS_PROD_${SERVER@U}"
### For ref and image tag generation: assume the 'inputs.ref' is a tag, and use the first two digits to construct the branch name
COMMIT_SHA="$(git rev-parse --verify HEAD)"
BRANCH=$(sed -E "s/v([0-9]+)\.([0-9]+)\..+$/release-\1.\2/g" <<<$REF)
SERVER_IMAGE_TAG="$(cut -c2- <<<$REF)" # Remove the leading 'v' from the given tag name, to generate the docker image tag
SERVER_IMAGE_ORG=mattermost
BUILD_ID_SUFFIX="release-${SERVER}-ent"
BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
WORKERS_NUMBER=20
TEST_FILTER_CYPRESS="${!_TEST_FILTER_CYPRESS_VARIABLE}"
TM4J_ENABLE=true
COMPUTED_REPORT_TYPE=RELEASE
### Run sanity assertions after variable generations
git show-ref --verify "refs/tags/${REF}" # 'inputs.ref' must be a tag, for release report types
git show-ref --verify "refs/remotes/origin/${BRANCH}" # The release branch computed from the given tag must exist
;;
*)
echo "Fatal: unimplemented test type. Aborting."
exit 1
esac
if [ -n "$ROLLING_RELEASE_FROM_TAG" ]; then
ROLLING_RELEASE_COMMIT_SHA=$(git rev-parse --verify "$ROLLING_RELEASE_FROM_TAG")
ROLLING_RELEASE_SERVER_IMAGE_TAG=$(echo "$ROLLING_RELEASE_FROM_TAG" | sed 's/^v//') # Remove the leading 'v' from the given tag name, to generate the docker image tag
ROLLING_RELEASE_SERVER_IMAGE="mattermost/mattermost-enterprise-edition:${ROLLING_RELEASE_SERVER_IMAGE_TAG}"
BUILD_ID_SUFFIX="rolling${ROLLING_RELEASE_FROM_TAG/-/_}-$BUILD_ID_SUFFIX"
BUILD_ID_SUFFIX_IN_STATUS_CHECK=true
WORKERS_NUMBER=10 # Rolling release tests are particularly impacted by increased parallelism. It's more efficient to run on fewer workers
### Run sanity assertions after variable generations
git show-ref --verify "refs/tags/${ROLLING_RELEASE_FROM_TAG}" # 'inputs.ROLLING_RELEASE_FROM_TAG' must be a tag, for release report types
fi
ENABLED_DOCKER_SERVICES="postgres inbucket minio openldap elasticsearch keycloak"
for SVC_OP in $(tr , ' '<<<"$MM_SERVICE_OVERRIDES"); do
OP=$(cut -c1 <<<$SVC_OP)
SVC=$(cut -c2- <<<$SVC_OP)
case "$OP" in
"+") ENABLED_DOCKER_SERVICES="$ENABLED_DOCKER_SERVICES $SVC" ;;
"-") ENABLED_DOCKER_SERVICES=$(sed -E "s:(^| )${SVC}( |\$): :g" <<<"$ENABLED_DOCKER_SERVICES") ;;
*) echo "Invalid MM_SERVICE_OVERRIDE value: $SVC_OP"; exit 1 ;;
esac
done
# Determine server image name and FIPS suffix based on FIPS_ENABLED parameter
if [ "$FIPS_ENABLED" = "true" ]; then
SERVER_IMAGE_NAME="mattermost-enterprise-fips-edition"
FIPS_SUFFIX="_fips"
else
SERVER_IMAGE_NAME="mattermost-enterprise-edition"
FIPS_SUFFIX=""
fi
# BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
# Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
BUILD_ID="${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}-${SERVER_IMAGE_TAG}${FIPS_SUFFIX}-${BUILD_ID_SUFFIX}"
echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
echo "BRANCH=${BRANCH}" >> $GITHUB_OUTPUT
echo "SERVER_IMAGE=${SERVER_IMAGE_ORG}/${SERVER_IMAGE_NAME}:${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
echo "FIPS_SUFFIX=${FIPS_SUFFIX}" >> $GITHUB_OUTPUT
echo "SERVER=${SERVER}" >> $GITHUB_OUTPUT
echo "server_uppercase=${SERVER@U}" >> $GITHUB_OUTPUT
echo "ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES}" >> $GITHUB_OUTPUT
echo "status_check_context=E2E Tests/test${FIPS_SUFFIX}${BUILD_ID_SUFFIX_IN_STATUS_CHECK:+-$BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT
echo "TEST_FILTER_CYPRESS=${TEST_FILTER_CYPRESS}" >> $GITHUB_OUTPUT
echo "TESTCASE_FAILURE_FATAL=${TESTCASE_FAILURE_FATAL}" >> $GITHUB_OUTPUT
echo "TM4J_ENABLE=${TM4J_ENABLE:-}" >> $GITHUB_OUTPUT
echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT
echo "ROLLING_RELEASE_commit_sha=${ROLLING_RELEASE_COMMIT_SHA}" >> $GITHUB_OUTPUT
echo "ROLLING_RELEASE_SERVER_IMAGE=${ROLLING_RELEASE_SERVER_IMAGE}" >> $GITHUB_OUTPUT
echo "BUILD_ID=${BUILD_ID}" >> $GITHUB_OUTPUT
# User notification variables
echo "WORKFLOW_RUN_URL=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" >> $GITHUB_OUTPUT
echo "CYCLE_URL=${AUTOMATION_DASHBOARD_URL%%/api}/cycle/${BUILD_ID}" >> $GITHUB_OUTPUT
- name: ci/notify-user
env:
COMMIT_SHA: "${{steps.generate.outputs.commit_sha}}"
STATUS_CHECK_CONTEXT: "${{steps.generate.outputs.status_check_context}}"
WORKFLOW_RUN_URL: "${{steps.generate.outputs.WORKFLOW_RUN_URL}}"
CYCLE_URL: "${{steps.generate.outputs.CYCLE_URL}}"
RUN_CYPRESS: "${{inputs.RUN_CYPRESS == 'true' || ''}}"
RUN_PLAYWRIGHT: "${{inputs.RUN_PLAYWRIGHT == 'true' || ''}}"
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_RUN_ID: ${{ github.run_id }}
run: |
if [ -n "$PR_NUMBER" ]; then
gh issue -R "${GITHUB_REPOSITORY}" comment "$PR_NUMBER" --body-file - <<EOF
E2E test run is starting for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}${MM_SERVICE_OVERRIDES:+, Cypress service overrides \`$MM_SERVICE_OVERRIDES\`}.
To check the run progress:
- Cypress: ${RUN_CYPRESS:+look for commit status \`$STATUS_CHECK_CONTEXT\` or the access the [Automation Dashboard Cycle URL]($CYCLE_URL)}$([ -n "${RUN_CYPRESS:-}" ] || echo -n "will not run").
- Playwright: ${RUN_PLAYWRIGHT:+look for commit status \`$STATUS_CHECK_CONTEXT-playwright\`}$([ -n "${RUN_PLAYWRIGHT:-}" ] || echo -n "will not run").
You can also look at the [E2E test's Workflow Run URL]($WORKFLOW_RUN_URL) (run ID \`${GITHUB_RUN_ID}\`).
EOF
fi
e2e-fulltest-cypress:
needs:
- generate-test-variables
uses: ./.github/workflows/e2e-tests-ci-template.yml
if: ${{ inputs.RUN_CYPRESS == 'true' }}
with:
commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}"
workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}"
testcase_failure_fatal: "${{ needs.generate-test-variables.outputs.TESTCASE_FAILURE_FATAL == 'true' }}"
enable_reporting: true
SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.SERVER_IMAGE }}"
ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}"
TEST: "cypress"
TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER_CYPRESS }}"
MM_ENV: "${{ inputs.MM_ENV || '' }}"
BRANCH: "${{ needs.generate-test-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
REPORT_TYPE: "${{ needs.generate-test-variables.outputs.REPORT_TYPE }}"
ROLLING_RELEASE_commit_sha: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_commit_sha }}"
ROLLING_RELEASE_SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
PR_NUMBER: "${{ inputs.PR_NUMBER }}"
secrets:
MM_LICENSE: "${{ secrets[format('MM_E2E_TEST_LICENSE_{0}_ENT', needs.generate-test-variables.outputs.server_uppercase)] }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}"
PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
REPORT_TM4J_API_KEY: "${{ needs.generate-test-variables.outputs.TM4J_ENABLE == 'true' && secrets.MM_E2E_TM4J_API_KEY || '' }}"
REPORT_TM4J_TEST_CYCLE_LINK_PREFIX: "${{ secrets.MM_E2E_TEST_CYCLE_LINK_PREFIX }}"
CWS_URL: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_URL || '' }}"
CWS_EXTRA_HTTP_HEADERS: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS || '' }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
e2e-fulltest-playwright:
needs:
- generate-test-variables
uses: ./.github/workflows/e2e-tests-ci-template.yml
if: ${{ inputs.RUN_PLAYWRIGHT == 'true' }}
with:
commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}-playwright"
workers_number: "1"
testcase_failure_fatal: "${{ needs.generate-test-variables.outputs.TESTCASE_FAILURE_FATAL == 'true' }}"
enable_reporting: true
SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.SERVER_IMAGE }}"
TEST: "playwright"
TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER_PLAYWRIGHT }}"
MM_ENV: "${{ inputs.MM_ENV || '' }}"
BRANCH: "${{ needs.generate-test-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
REPORT_TYPE: "${{ needs.generate-test-variables.outputs.REPORT_TYPE }}"
ROLLING_RELEASE_commit_sha: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_commit_sha }}"
ROLLING_RELEASE_SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.ROLLING_RELEASE_SERVER_IMAGE }}"
PR_NUMBER: "${{ inputs.PR_NUMBER }}"
secrets:
MM_LICENSE: "${{ secrets[format('MM_E2E_TEST_LICENSE_{0}_ENT', needs.generate-test-variables.outputs.server_uppercase)] }}"
PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
CWS_URL: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_URL || '' }}"
CWS_EXTRA_HTTP_HEADERS: "${{ needs.generate-test-variables.outputs.SERVER == 'cloud' && secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS || '' }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
notify-user:
runs-on: ubuntu-24.04
if: always()
needs:
- generate-test-variables
- e2e-fulltest-cypress
- e2e-fulltest-playwright
permissions:
issues: write
pull-requests: write
defaults:
run:
shell: bash
env:
GH_TOKEN: "${{ github.token }}"
PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
MM_ENV: "${{ inputs.MM_ENV || '' }}"
COMMIT_SHA: "${{ needs.generate-test-variables.outputs.commit_sha }}"
STATUS_CHECK_CONTEXT: "${{ needs.generate-test-variables.outputs.status_check_context }}"
WORKFLOW_RUN_URL: "${{ needs.generate-test-variables.outputs.WORKFLOW_RUN_URL }}"
CYCLE_URL: "${{ needs.generate-test-variables.outputs.CYCLE_URL }}"
RUN_CYPRESS: "${{inputs.RUN_CYPRESS == 'true' || ''}}"
RUN_PLAYWRIGHT: "${{inputs.RUN_PLAYWRIGHT == 'true' || ''}}"
PLAYWRIGHT_REPORT_URL: "${{ needs.e2e-fulltest-playwright.outputs.playwright_report_url }}"
steps:
- name: ci/notify-user-test-completion
env:
GITHUB_REPOSITORY: ${{ github.repository }}
CYPRESS_PASS_RATE: ${{ needs.e2e-fulltest-cypress.outputs.pass_rate }}
PLAYWRIGHT_PASS_RATE: ${{ needs.e2e-fulltest-playwright.outputs.pass_rate }}
run: |
if [ -n "$PR_NUMBER" ]; then
gh issue -R "${GITHUB_REPOSITORY}" comment "$PR_NUMBER" --body-file - <<EOF
E2E test has completed for commit \`${COMMIT_SHA}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
Results summary:
- Cypress: ${RUN_CYPRESS:+pass rate is \`${CYPRESS_PASS_RATE:-unknown}\` (see [Automation Dashboard]($CYCLE_URL) and commit status check \`$STATUS_CHECK_CONTEXT\`)}$([ -n "${RUN_CYPRESS:-}" ] || echo -n "did not run").
- Playwright: ${RUN_PLAYWRIGHT:+pass rate is \`${PLAYWRIGHT_PASS_RATE:-unknown}\` (see [Playwright Report URL]($PLAYWRIGHT_REPORT_URL) and commit status check \`$STATUS_CHECK_CONTEXT-playwright\`)}$([ -n "${RUN_PLAYWRIGHT:-}" ] || echo -n "did not run").
The run summary artifacts are available in the corresponding [Workflow Run]($WORKFLOW_RUN_URL).
EOF
fi
-542
View File
@@ -1,542 +0,0 @@
---
name: E2E Tests Template
on:
workflow_call:
inputs:
# NB: this does not support using branch names that belong to forks.
# In those cases, you should specify directly the commit SHA that you want to test, or
# some wrapper workflow that does it for you (e.g. the slash command for initiating a PR test)
commit_sha:
type: string
required: true
status_check_context:
type: string
required: true
workers_number:
type: string # Should ideally be a number; see https://github.com/orgs/community/discussions/67182
required: false
default: "1"
testcase_failure_fatal:
type: boolean
required: false
default: true
enable_reporting:
type: boolean
required: false
default: false
SERVER:
type: string # Valid values are: onprem, cloud
required: false
default: onprem
SERVER_IMAGE:
type: string
required: false
ENABLED_DOCKER_SERVICES:
type: string
required: false
TEST: # Valid values are: cypress, playwright
type: string
required: false
default: "cypress"
TEST_FILTER:
type: string
required: false
MM_ENV:
type: string
required: false
BRANCH:
type: string
required: false
BUILD_ID:
type: string
required: false
REPORT_TYPE:
type: string
required: false
ROLLING_RELEASE_commit_sha:
type: string
required: false
ROLLING_RELEASE_SERVER_IMAGE:
type: string
required: false
PR_NUMBER:
type: string
required: false
secrets:
MM_LICENSE:
required: false
AUTOMATION_DASHBOARD_URL:
required: false
AUTOMATION_DASHBOARD_TOKEN:
required: false
PUSH_NOTIFICATION_SERVER:
required: false
REPORT_WEBHOOK_URL:
required: false
REPORT_TM4J_API_KEY:
required: false
REPORT_TM4J_TEST_CYCLE_LINK_PREFIX:
required: false
CWS_URL:
required: false
CWS_EXTRA_HTTP_HEADERS:
required: false
AWS_ACCESS_KEY_ID:
required: false
AWS_SECRET_ACCESS_KEY:
required: false
outputs:
passed:
value: "${{ jobs.report.outputs.passed }}"
failed:
value: "${{ jobs.report.outputs.failed }}"
failed_expected:
value: "${{ jobs.report.outputs.failed_expected }}"
pass_rate:
value: "${{ jobs.report.outputs.pass_rate }}"
playwright_report_url:
value: ${{ jobs.report.outputs.playwright_report_url }}
permissions: {}
jobs:
update-initial-status:
permissions:
statuses: write
runs-on: ubuntu-24.04
steps:
- uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: E2E tests for mattermost server app
status: pending
generate-build-variables:
permissions:
contents: read
runs-on: ubuntu-24.04
needs:
- update-initial-status
defaults:
run:
shell: bash
outputs:
workers: "${{ steps.generate.outputs.workers }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/generate-build-variables
id: generate
env:
WORKERS: ${{ inputs.workers_number }}
run: |
[ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0
echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT
generate-test-cycle:
permissions:
contents: read
runs-on: ubuntu-24.04
needs:
- generate-build-variables
defaults:
run:
shell: bash
working-directory: e2e-tests
outputs:
status_check_url: "${{ steps.e2e-test-gencycle.outputs.status_check_url }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
id: setup_node
with:
node-version-file: ".nvmrc"
- name: ci/e2e-test-gencycle
id: e2e-test-gencycle
env:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
BRANCH: "${{ inputs.BRANCH }}"
BUILD_ID: "${{ inputs.BUILD_ID }}"
TEST: "${{ inputs.TEST }}"
TEST_FILTER: "${{ inputs.TEST_FILTER }}"
GITHUB_SERVER_URL: ${{ github.server_url }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_RUN_ID: ${{ github.run_id }}
run: |
set -e -o pipefail
make generate-test-cycle | tee generate-test-cycle.out
# Extract cycle's dashboard URL, if present
TEST_CYCLE_ID=$(sed -nE "s/^.*id: '([^']+)'.*$/\1/p" <generate-test-cycle.out)
if [ -n "$TEST_CYCLE_ID" ]; then
echo "status_check_url=https://automation-dashboard.vercel.app/cycles/${TEST_CYCLE_ID}" >> $GITHUB_OUTPUT
else
echo "status_check_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" >> $GITHUB_OUTPUT
fi
test:
permissions:
contents: read
actions: write
continue-on-error: true # Individual runner failures shouldn't prevent the completion of an E2E run
strategy:
fail-fast: false # Individual runner failures shouldn't prevent the completion of an E2E run
matrix:
#
# Note that E2E tests should be run only on ubuntu, for QA purposes.
# But it's useful to be able to run and debug the E2E tests for different OSes.
# Notes:
# - For MacOS: works on developer machines, but uses too many resources to be able to run on Github Actions
# - for Windows: cannot currently run on Github Actions, since the runners do not support running linux containers, at the moment
#
#os: [ubuntu-24.04, windows-2022, macos-12-xl]
os: [ubuntu-24.04]
worker_index: ${{ fromJSON(needs.generate-build-variables.outputs.workers) }} # https://docs.github.com/en/actions/learn-github-actions/expressions#example-returning-a-json-object
runs-on: "${{ matrix.os }}"
timeout-minutes: 120
needs:
- generate-build-variables
- generate-test-cycle
defaults:
run:
shell: bash
working-directory: e2e-tests
env:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
SERVER: "${{ inputs.SERVER }}"
SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
MM_LICENSE: "${{ secrets.MM_LICENSE }}"
ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}"
TEST: "${{ inputs.TEST }}"
TEST_FILTER: "${{ inputs.TEST_FILTER }}"
MM_ENV: "${{ inputs.MM_ENV }}"
BRANCH: "${{ inputs.BRANCH }}"
BUILD_ID: "${{ inputs.BUILD_ID }}"
CI_BASE_URL: "${{ matrix.os }}-${{ matrix.worker_index }}"
CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
CWS_URL: "${{ secrets.CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}"
ROLLING_RELEASE_COMMIT_SHA: "${{ inputs.ROLLING_RELEASE_commit_sha }}"
ROLLING_RELEASE_SERVER_IMAGE: "${{ inputs.ROLLING_RELEASE_SERVER_IMAGE }}"
steps:
- name: ci/checkout-actions
# Sparse-checkout just .github/actions from the triggering ref (master)
# so the composite action below is available before the full checkout
# overwrites the workspace with inputs.commit_sha.
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
sparse-checkout: .github/actions
sparse-checkout-cone-mode: true
- name: ci/runner-prep-for-openldap
uses: ./.github/actions/runner-prep-openldap
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-macos-docker
if: runner.os == 'macos'
# https://github.com/actions/runner-images/issues/17#issuecomment-1537238473
run: |
brew install docker docker-compose
colima start
mkdir -p ~/.docker/cli-plugins
ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
id: setup_node
with:
node-version-file: ".nvmrc"
- name: ci/e2e-test
run: |
make cloud-init
if [ -n "$ROLLING_RELEASE_SERVER_IMAGE" ]; then
echo "RollingRelease: checking out E2E test cases from revision ${ROLLING_RELEASE_COMMIT_SHA}, for initial smoketest"
git checkout "${ROLLING_RELEASE_COMMIT_SHA}" -- "${TEST}/" && git status
(
echo "RollingRelease: running initial smoketest against image $ROLLING_RELEASE_SERVER_IMAGE"
export SERVER_IMAGE="$ROLLING_RELEASE_SERVER_IMAGE"
export TEST_FILTER=""
export AUTOMATION_DASHBOARD_URL=""
make
)
echo "RollingRelease: asserting smoketest result has zero failures."
FAILURES=$(jq -r '.failed' "${TEST}/results/summary.json")
if [ "$FAILURES" -ne "0" ]; then
echo "RollingRelease: initial smoketest for rolling release E2E run has nonzero ($FAILURES) failures. Aborting test run." >&2
exit 1
fi
rm -rfv "${TEST}/{results,logs}"
echo "RollingRelease: reset the E2E test cases to the revision to test"
git reset --hard HEAD && git status
echo "RollingRelease: smoketest completed. Starting full E2E tests."
fi
make
- name: ci/cloud-teardown
if: always()
run: make cloud-teardown
- name: ci/dump-docker-state-on-failure
# Always run a final docker-state capture so failures unrelated to
# openldap startup (e.g. server container later crashes) still produce
# logs we can inspect. The script's own retry loop dumps openldap
# state per-attempt; this step is a backstop covering the whole job.
if: failure()
run: |
set +e
DIAG="e2e-tests/docker-diagnostics/job-failure"
mkdir -p "$DIAG"
docker ps -a >"$DIAG/docker.ps.txt" 2>&1
docker version >"$DIAG/docker.version.txt" 2>&1
docker info >"$DIAG/docker.info.txt" 2>&1
for c in $(docker ps -a --format '{{.Names}}'); do
docker inspect "$c" >"$DIAG/$c.inspect.json" 2>&1
docker logs "$c" >"$DIAG/$c.log" 2>&1
done
uname -a >"$DIAG/host.uname.txt" 2>&1
free -m >"$DIAG/host.free.txt" 2>&1
df -h >"$DIAG/host.df.txt" 2>&1
sudo dmesg | tail -500 >"$DIAG/host.dmesg.tail.txt" 2>&1
sudo dmesg | grep -iE 'apparmor|denied|oom|killed|openldap|slapd' >"$DIAG/host.dmesg.relevant.txt" 2>&1
- name: ci/upload-docker-diagnostics
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
with:
name: docker-diagnostics-${{ inputs.TEST }}-${{ matrix.os }}-${{ matrix.worker_index }}
path: e2e-tests/docker-diagnostics/
retention-days: 7
if-no-files-found: ignore
- name: ci/e2e-test-store-results
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
with:
name: e2e-test-results-${{ inputs.TEST }}-${{ matrix.os }}-${{ matrix.worker_index }}
path: |
e2e-tests/${{ inputs.TEST }}/logs/
e2e-tests/${{ inputs.TEST }}/results/
retention-days: 1
report:
permissions:
actions: write
contents: read
runs-on: ubuntu-24.04
needs:
- test
- generate-build-variables
defaults:
run:
shell: bash
working-directory: e2e-tests
outputs:
passed: "${{ steps.calculate-results.outputs.passed }}"
failed: "${{ steps.calculate-results.outputs.failed }}"
failed_expected: "${{ steps.calculate-results.outputs.failed_expected }}"
pass_rate: "${{ steps.calculate-results.outputs.pass_rate }}"
commit_status_message: "${{ steps.calculate-results.outputs.commit_status_message }}"
playwright_report_url: "${{ steps.upload-to-s3.outputs.report_url }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/download-artifacts
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
pattern: e2e-test-results-${{ inputs.TEST }}-*
path: e2e-tests/${{ inputs.TEST }}/
merge-multiple: true
- name: ci/upload-report-global
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: e2e-test-results-${{ inputs.TEST }}
path: |
e2e-tests/${{ inputs.TEST }}/logs/
e2e-tests/${{ inputs.TEST }}/results/
- name: ci/restore-npm-cache
if: "${{ inputs.enable_reporting }}"
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
if: "${{ inputs.enable_reporting }}"
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
id: setup_node
with:
node-version-file: ".nvmrc"
- name: ci/publish-report
if: "${{ inputs.enable_reporting }}"
env:
TYPE: "${{ inputs.REPORT_TYPE }}"
TEST: "${{ inputs.TEST }}"
SERVER: "${{ inputs.SERVER }}"
SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}"
PR_NUMBER: "${{ inputs.PR_NUMBER }}"
BRANCH: "${{ inputs.BRANCH }}"
BUILD_ID: "${{ inputs.BUILD_ID }}"
MM_ENV: "${{ inputs.MM_ENV }}"
TM4J_API_KEY: "${{ secrets.REPORT_TM4J_API_KEY }}"
TEST_CYCLE_LINK_PREFIX: "${{ secrets.REPORT_TM4J_TEST_CYCLE_LINK_PREFIX }}"
run: |
echo "DEBUG: TYPE=${TYPE}, PR_NUMBER=${PR_NUMBER:-<not set>}"
make report
# The results dir may have been modified as part of the reporting: re-upload
- name: ci/upload-report-global
if: "${{ inputs.enable_reporting }}"
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: e2e-test-results-${{ inputs.TEST }}
path: |
e2e-tests/${{ inputs.TEST }}/logs/
e2e-tests/${{ inputs.TEST }}/results/
overwrite: true
# Configure AWS credentials
- name: ci/aws-configure
if: (inputs.TEST == 'playwright')
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
aws-region: us-east-1
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# Upload the playwright reports to S3
- name: ci/upload-results-to-s3
if: (inputs.TEST == 'playwright')
id: upload-to-s3
env:
AWS_REGION: us-east-1
AWS_S3_BUCKET: mattermost-cypress-report
INPUT_PR_NUMBER: ${{ inputs.PR_NUMBER }}
INPUT_TEST: ${{ inputs.TEST }}
GITHUB_RUN_ID: ${{ github.run_id }}
run: |
echo "🔍 Checking if results directory exists..."
LOCAL_RESULTS_PATH="${INPUT_TEST}/results/"
LOCAL_LOGS_PATH="${INPUT_TEST}/logs/"
S3_PATH="server-pr-${INPUT_PR_NUMBER}/e2e-reports/${INPUT_TEST}/${GITHUB_RUN_ID}"
echo "📤 Uploading to s3://${AWS_S3_BUCKET}/${S3_PATH}/"
if [[ -d "$LOCAL_RESULTS_PATH" ]]; then
aws s3 sync "$LOCAL_RESULTS_PATH" "s3://${AWS_S3_BUCKET}/${S3_PATH}/results/" \
--acl public-read \
--cache-control "no-cache"
fi
REPORT_URL="https://${AWS_S3_BUCKET}.s3.amazonaws.com/${S3_PATH}/results/reporter/index.html"
echo "✅ Report uploaded to: $REPORT_URL"
echo "report_url=$REPORT_URL" >> "$GITHUB_OUTPUT"
- name: ci/report-calculate-results
id: calculate-results
env:
TEST: "${{ inputs.TEST }}"
run: |
AD_CYCLE_FILE="${TEST}/results/ad_cycle.json"
if [ -f "$AD_CYCLE_FILE" ]; then
# Prefer using the Automation Dashboard's results to calculate failures
export PASSED=$(jq -r .pass "$AD_CYCLE_FILE")
export FAILED=$(jq -r .fail "$AD_CYCLE_FILE")
export FAILED_EXPECTED=$(jq -r ".known + .flaky + .skipped" "$AD_CYCLE_FILE")
else
# Otherwise, utilize summary.json to calculate the failures
# NB: in this job, this file only makes sense if a single worker is used, as with Playwright
export PASSED=$(jq '.passed' "${TEST}/results/summary.json")
export FAILED=$(jq '.failed' "${TEST}/results/summary.json")
export FAILED_EXPECTED=$(jq '.failed_expected' "${TEST}/results/summary.json")
fi
export TOTAL_SPECS=$(( PASSED + FAILED ))
export PASS_RATE=$(jq -r '100 * (env.PASSED | tonumber) / (env.TOTAL_SPECS | tonumber)' <<<'{}' | xargs -l printf '%.2f')
if [ "$FAILED" = "0" ]; then
export COMMIT_STATUS_MESSAGE="All test cases passed"
else
export COMMIT_STATUS_MESSAGE="${FAILED} test cases failed. Please check the workflow logs"
fi
echo "passed=${PASSED:?}" >> $GITHUB_OUTPUT
echo "failed=${FAILED:?}" >> $GITHUB_OUTPUT
echo "failed_expected=${FAILED_EXPECTED:?}" >> $GITHUB_OUTPUT
echo "pass_rate=${PASS_RATE:?}%" >> $GITHUB_OUTPUT
echo "commit_status_message=${COMMIT_STATUS_MESSAGE:?}" >> $GITHUB_OUTPUT
echo "$COMMIT_STATUS_MESSAGE"
- name: ci/e2e-test-assert-results
if: "${{ inputs.testcase_failure_fatal }}"
env:
CALCULATE_FAILED: ${{ steps.calculate-results.outputs.failed }}
run: |
# Assert that the run contained 0 failures
[ "${CALCULATE_FAILED}" = "0" ]
update-failure-final-status:
permissions:
statuses: write
runs-on: ubuntu-24.04
if: failure() || cancelled()
needs:
- generate-test-cycle
- test
- report
steps:
- uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: ${{ needs.report.outputs.commit_status_message || 'Error during test execution' }}
status: failure
target_url: >-
${{ inputs.TEST == 'playwright'
&& needs.report.outputs.playwright_report_url
|| needs.generate-test-cycle.outputs.status_check_url }}
update-success-final-status:
permissions:
statuses: write
runs-on: ubuntu-24.04
if: success()
needs:
- generate-test-cycle
- test
- report
steps:
- uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: ${{ needs.report.outputs.commit_status_message || 'Error during test execution' }}
status: success
target_url: >-
${{ inputs.TEST == 'playwright'
&& needs.report.outputs.playwright_report_url
|| needs.generate-test-cycle.outputs.status_check_url }}
-4
View File
@@ -261,8 +261,6 @@ jobs:
should_run: "${{ needs.check-changes.outputs.should_run }}"
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
# Enterprise FIPS Edition
@@ -315,6 +313,4 @@ jobs:
pr_number: "${{ needs.resolve-pr.outputs.PR_NUMBER }}"
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
@@ -1,404 +0,0 @@
---
name: E2E Tests - Cypress Template (v2 - test system io dispatch)
# Delegates Cypress spec dispatch + reporting to test system io.
# Authenticates via GitHub Actions OIDC; calling job MUST grant
# `id-token: write`.
on:
workflow_call:
inputs:
test_type:
description: "Type of test run (smoke or full)"
type: string
required: true
workers:
description: "Number of parallel test system io dispatch workers"
type: number
required: false
default: 40
enabled_docker_services:
description: "Space-separated list of docker services to enable"
type: string
required: false
default: "postgres inbucket minio openldap elasticsearch keycloak"
commit_sha:
type: string
required: true
branch:
type: string
required: true
build_id:
type: string
required: true
server_image_tag:
description: "Server image tag (e.g., master or short SHA)"
type: string
required: true
server:
type: string
required: false
default: onprem
server_edition:
description: "Server edition: enterprise (default), fips, or team"
type: string
required: false
default: enterprise
server_image_repo:
description: "Docker registry: mattermostdevelopment (default) or mattermost"
type: string
required: false
default: mattermostdevelopment
server_image_aliases:
description: "Comma-separated alias tags for description"
type: string
required: false
enable_reporting:
type: boolean
required: false
default: false
report_type:
type: string
required: false
ref_branch:
type: string
required: false
pr_number:
type: string
required: false
context_name:
description: "GitHub commit status context name"
type: string
required: true
cypress_stage:
description: "Comma-separated `// Stage:` tags; spec must share at least one. Empty disables filter."
type: string
required: false
default: "@prod"
cypress_include_group:
description: "Comma-separated `// Group:` tags; spec must share at least one. Empty disables filter."
type: string
required: false
default: ""
cypress_exclude_group:
description: "Comma-separated `// Group:` tags; spec dropped if it shares any."
type: string
required: false
default: "@te_only,@cloud_only,@high_availability"
cypress_skip_on:
description: "Comma-separated active-env tag(s); spec dropped if its `// Skip:` line shares any."
type: string
required: false
default: "@headless"
cypress_sort_first:
description: "Comma-separated `// Group:` tags; matching specs dispatch first."
type: string
required: false
default: "@compliance_export,@elasticsearch,@ldap_group,@ldap"
cypress_sort_last:
description: "Comma-separated `// Group:` tags; matching specs dispatch last."
type: string
required: false
default: "@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
retest_on_fail:
description: "Re-dispatch failed dispatch units once (whole-spec retry, on top of cypress.config retries)"
type: boolean
required: false
default: true
secrets:
MM_LICENSE:
required: false
AUTOMATION_DASHBOARD_URL:
required: false
AUTOMATION_DASHBOARD_TOKEN:
required: false
PUSH_NOTIFICATION_SERVER:
required: false
REPORT_WEBHOOK_URL:
required: false
CWS_URL:
required: false
CWS_EXTRA_HTTP_HEADERS:
required: false
# Callers must grant: contents: read, statuses: write, id-token: write
permissions:
contents: read
statuses: write
id-token: write
env:
SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}"
jobs:
prepare-run:
runs-on: ubuntu-24.04
permissions:
contents: read
id-token: write
statuses: write
outputs:
composite-identity-json: ${{ steps.composite-identity.outputs.composite-identity-json }}
workers-matrix: ${{ steps.matrix.outputs.workers }}
start_time: ${{ steps.matrix.outputs.start_time }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/composite-identity
id: composite-identity
env:
CONTEXT_NAME: ${{ inputs.context_name }}
GITHUB_REPOSITORY: ${{ github.repository }}
MM_BRANCH: ${{ inputs.branch }}
MM_SHA: ${{ inputs.commit_sha }}
PR_NUMBER: ${{ inputs.pr_number }}
run: |
# Derive the test-system-io run name from the GitHub commit-status
# context: drop the `e2e-test/` prefix (the framework name already
# implies E2E in the dashboard) and swap remaining `/` for `-` so
# the dashboard URL is path-safe. The commit-status context itself
# stays unchanged elsewhere — branch protection rules depend on it.
NAME="${CONTEXT_NAME#e2e-test/}"
NAME="${NAME//\//-}"
# gh_pr_number is optional; include it only when present.
if [ -n "$PR_NUMBER" ]; then
COMPOSITE_IDENTITY=$(jq -nc \
--arg repo "${GITHUB_REPOSITORY}" \
--arg sha "${MM_SHA}" \
--arg run_id "${GITHUB_RUN_ID}" \
--arg name "${NAME}" \
--arg attempt "${GITHUB_RUN_ATTEMPT}" \
--arg branch "${MM_BRANCH}" \
--arg pr "${PR_NUMBER}" \
'{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}')
else
COMPOSITE_IDENTITY=$(jq -nc \
--arg repo "${GITHUB_REPOSITORY}" \
--arg sha "${MM_SHA}" \
--arg run_id "${GITHUB_RUN_ID}" \
--arg name "${NAME}" \
--arg attempt "${GITHUB_RUN_ATTEMPT}" \
--arg branch "${MM_BRANCH}" \
'{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}')
fi
echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> $GITHUB_OUTPUT
- name: ci/matrix
id: matrix
env:
INPUT_WORKERS: ${{ inputs.workers }}
run: |
echo "workers=$(jq -nc --argjson n "${INPUT_WORKERS}" '[range(1; $n+1)]')" >> $GITHUB_OUTPUT
echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
# Install webapp node_modules once via the shared webapp-setup action, then
# workers restore the same stable cache. The node_modules cache is keyed only
# on webapp/package-lock.json and is shared with webapp-ci.yml jobs.
prep-deps:
name: prep-deps
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
contents: read
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/setup-webapp-node-modules
uses: ./.github/actions/webapp-setup
- name: ci/cache-cypress-deps
# node_modules + the cypress binary (downloaded to ~/.cache/Cypress by
# cypress's postinstall, not into node_modules). Both must be cached;
# otherwise workers see "cypress npm package installed but binary missing".
id: cache-cypress
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: |
e2e-tests/cypress/node_modules
~/.cache/Cypress
key: e2e-cypress-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/cypress/package-lock.json') }}
- name: ci/install-cypress-deps
if: steps.cache-cypress.outputs.cache-hit != 'true'
working-directory: e2e-tests/cypress
run: npm ci
# Register the Test System IO run AFTER prep-deps so workers reach
# dispatch-run within Test System IO's inactivity window.
dispatch-begin:
runs-on: ubuntu-24.04
needs: [prepare-run, prep-deps]
permissions:
contents: read
id-token: write
statuses: write
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/dispatch-begin
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-begin@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
framework: cypress
repo-dir: ${{ github.workspace }}
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
total-reports-expected: ${{ inputs.workers }}
retest-on-fail: ${{ inputs.retest_on_fail }}
cypress-stage: ${{ inputs.cypress_stage }}
cypress-include-group: ${{ inputs.cypress_include_group }}
cypress-exclude-group: ${{ inputs.cypress_exclude_group }}
cypress-skip-on: ${{ inputs.cypress_skip_on }}
cypress-sort-first: ${{ inputs.cypress_sort_first }}
cypress-sort-last: ${{ inputs.cypress_sort_last }}
github-token: ${{ secrets.GITHUB_TOKEN }}
commit-status-context: ${{ inputs.context_name }}
image-tag: ${{ inputs.server_image_tag }}
image-aliases: ${{ inputs.server_image_aliases }}
workers:
name: dispatch-run-${{ matrix.worker_index }}
runs-on: ubuntu-24.04
timeout-minutes: 30
needs: [prepare-run, dispatch-begin]
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
matrix:
worker_index: ${{ fromJSON(needs.prepare-run.outputs.workers-matrix) }}
env:
COMPOSITE_IDENTITY: ${{ needs.prepare-run.outputs.composite-identity-json }}
SERVER: "${{ inputs.server }}"
MM_LICENSE: "${{ secrets.MM_LICENSE }}"
ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}"
TEST: cypress
# The dispatch adapter invokes `npx cypress run` directly (no
# cross-env), so pin TZ here — several specs assume UTC.
TZ: Etc/UTC
BRANCH: "${{ inputs.branch }}"
BUILD_ID: "${{ inputs.build_id }}"
CI_BASE_URL: "${{ inputs.test_type }}-test-${{ matrix.worker_index }}"
CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
CWS_URL: "${{ secrets.CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}"
steps:
- name: ci/checkout-actions
# Sparse-checkout just .github/actions from the triggering ref (master)
# so the composite action below is available before the full checkout
# overwrites the workspace with inputs.commit_sha.
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
sparse-checkout: .github/actions
sparse-checkout-cone-mode: true
- name: ci/runner-prep-for-openldap
uses: ./.github/actions/runner-prep-openldap
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-webapp-node-modules
uses: ./.github/actions/webapp-setup
with:
read-only: "true"
- name: ci/restore-cypress-deps
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: |
e2e-tests/cypress/node_modules
~/.cache/Cypress
key: e2e-cypress-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/cypress/package-lock.json') }}
fail-on-cache-miss: true
- name: ci/cloud-init
working-directory: e2e-tests
run: make cloud-init
- name: ci/start-server
working-directory: e2e-tests
run: make start-server
- name: ci/dispatch-run
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-run@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
framework: cypress
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
repo-dir: ${{ github.workspace }}
artifacts-root: ${{ github.workspace }}/worker-artifacts
github-token: ${{ secrets.GITHUB_TOKEN }}
gh-job-name: dispatch-run-${{ matrix.worker_index }}
- name: ci/cloud-teardown
if: always()
working-directory: e2e-tests
run: make cloud-teardown
- name: ci/upload-debug-artifacts
if: always()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-debug-${{ matrix.worker_index }}
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
e2e-tests/cypress/tests/screenshots/
worker-artifacts/
retention-days: 5
if-no-files-found: ignore
report:
runs-on: ubuntu-24.04
needs: [prepare-run, dispatch-begin, workers]
if: always()
permissions:
contents: read
id-token: write
statuses: write
outputs:
commit_status_description: ${{ steps.summary.outputs.commit_status_description }}
webhook_payload: ${{ steps.summary.outputs.webhook_payload }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: ci/run-summary
id: summary
continue-on-error: true
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
framework: cypress
report-type: ${{ inputs.report_type }}
image-tag: ${{ inputs.server_image_tag }}
image-aliases: ${{ inputs.server_image_aliases }}
server-image: ${{ env.SERVER_IMAGE }}
pr-number: ${{ inputs.pr_number }}
ref-branch: ${{ inputs.ref_branch }}
commit-status-context: ${{ inputs.context_name }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: ci/publish-webhook
if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != ''
env:
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
PAYLOAD: ${{ steps.summary.outputs.webhook_payload }}
run: |
curl -X POST -H "Content-Type: application/json" -d "$PAYLOAD" "$REPORT_WEBHOOK_URL"
- name: ci/assert-results
env:
SUMMARY_OUTCOME: ${{ steps.summary.outcome }}
run: |
[ "$SUMMARY_OUTCOME" = "success" ]
+243 -562
View File
@@ -1,29 +1,28 @@
---
name: E2E Tests - Cypress Template
name: E2E Tests - Cypress Template (test system io dispatch)
# Delegates Cypress spec dispatch + reporting to test system io.
# Authenticates via GitHub Actions OIDC; calling job MUST grant
# `id-token: write`.
on:
workflow_call:
inputs:
# Test configuration
test_type:
description: "Type of test run (smoke or full)"
type: string
required: true
test_filter:
description: "Test filter arguments"
type: string
required: true
workers:
description: "Number of parallel workers"
description: "Number of parallel test system io dispatch workers"
type: number
required: false
default: 1
default: 40
enabled_docker_services:
description: "Space-separated list of docker services to enable"
type: string
required: false
default: "postgres inbucket"
default: "postgres inbucket minio openldap elasticsearch keycloak"
# Common build variables
commit_sha:
type: string
required: true
@@ -52,11 +51,10 @@ on:
required: false
default: mattermostdevelopment
server_image_aliases:
description: "Comma-separated alias tags for description (e.g., 'release-11.4, release-11')"
description: "Comma-separated alias tags for description"
type: string
required: false
# Reporting options
enable_reporting:
type: boolean
required: false
@@ -65,28 +63,51 @@ on:
type: string
required: false
ref_branch:
description: "Source branch name for webhook messages (e.g., 'master' or 'release-11.4')"
type: string
required: false
pr_number:
type: string
required: false
# Commit status configuration
context_name:
description: "GitHub commit status context name"
type: string
required: true
outputs:
passed:
description: "Number of passed tests"
value: ${{ jobs.report.outputs.passed }}
failed:
description: "Number of failed tests"
value: ${{ jobs.report.outputs.failed }}
status_check_url:
description: "URL to test results"
value: ${{ jobs.generate-test-cycle.outputs.status_check_url }}
cypress_stage:
description: "Comma-separated `// Stage:` tags; spec must share at least one. Empty disables filter."
type: string
required: false
default: "@prod"
cypress_include_group:
description: "Comma-separated `// Group:` tags; spec must share at least one. Empty disables filter."
type: string
required: false
default: ""
cypress_exclude_group:
description: "Comma-separated `// Group:` tags; spec dropped if it shares any."
type: string
required: false
default: "@te_only,@cloud_only,@high_availability"
cypress_skip_on:
description: "Comma-separated active-env tag(s); spec dropped if its `// Skip:` line shares any."
type: string
required: false
default: "@headless"
cypress_sort_first:
description: "Comma-separated `// Group:` tags; matching specs dispatch first."
type: string
required: false
default: "@compliance_export,@elasticsearch,@ldap_group,@ldap"
cypress_sort_last:
description: "Comma-separated `// Group:` tags; matching specs dispatch last."
type: string
required: false
default: "@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
retest_on_fail:
description: "Re-dispatch failed dispatch units once (whole-spec retry, on top of cypress.config retries)"
type: boolean
required: false
default: true
secrets:
MM_LICENSE:
@@ -104,620 +125,280 @@ on:
CWS_EXTRA_HTTP_HEADERS:
required: false
# Callers must grant at least these scopes on the job that uses this workflow
# Callers must grant: contents: read, statuses: write, id-token: write
permissions:
contents: read
statuses: write
id-token: write
env:
SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}"
jobs:
update-initial-status:
prepare-run:
runs-on: ubuntu-24.04
permissions:
contents: read
id-token: write
statuses: write
steps:
- name: ci/set-initial-status
uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "tests running, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: pending
generate-test-cycle:
runs-on: ubuntu-24.04
outputs:
status_check_url: "${{ steps.generate-cycle.outputs.status_check_url }}"
workers: "${{ steps.generate-workers.outputs.workers }}"
start_time: "${{ steps.generate-workers.outputs.start_time }}"
composite-identity-json: ${{ steps.composite-identity.outputs.composite-identity-json }}
workers-matrix: ${{ steps.matrix.outputs.workers }}
start_time: ${{ steps.matrix.outputs.start_time }}
steps:
- name: ci/generate-workers
id: generate-workers
env:
INPUT_WORKERS: ${{ inputs.workers }}
run: |
echo "workers=$(jq -nc --argjson n "${INPUT_WORKERS}" '[range($n)]')" >> $GITHUB_OUTPUT
echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version-file: ".nvmrc"
- name: ci/generate-test-cycle
id: generate-cycle
working-directory: e2e-tests
fetch-depth: 1
- name: ci/composite-identity
id: composite-identity
env:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
BRANCH: "${{ inputs.branch }}-${{ inputs.test_type }}"
BUILD_ID: "${{ inputs.build_id }}"
TEST: cypress
TEST_FILTER: "${{ inputs.test_filter }}"
GITHUB_SERVER_URL: ${{ github.server_url }}
CONTEXT_NAME: ${{ inputs.context_name }}
GITHUB_REPOSITORY: ${{ github.repository }}
GITHUB_RUN_ID: ${{ github.run_id }}
MM_BRANCH: ${{ inputs.branch }}
MM_SHA: ${{ inputs.commit_sha }}
PR_NUMBER: ${{ inputs.pr_number }}
run: |
set -e -o pipefail
make generate-test-cycle | tee generate-test-cycle.out
TEST_CYCLE_ID=$(sed -nE "s/^.*id: '([^']+)'.*$/\1/p" <generate-test-cycle.out)
if [ -n "$TEST_CYCLE_ID" ]; then
echo "status_check_url=https://automation-dashboard.vercel.app/cycles/${TEST_CYCLE_ID}" >> $GITHUB_OUTPUT
# Derive the test-system-io run name from the GitHub commit-status
# context: drop the `e2e-test/` prefix (the framework name already
# implies E2E in the dashboard) and swap remaining `/` for `-` so
# the dashboard URL is path-safe. The commit-status context itself
# stays unchanged elsewhere — branch protection rules depend on it.
NAME="${CONTEXT_NAME#e2e-test/}"
NAME="${NAME//\//-}"
# gh_pr_number is optional; include it only when present.
if [ -n "$PR_NUMBER" ]; then
COMPOSITE_IDENTITY=$(jq -nc \
--arg repo "${GITHUB_REPOSITORY}" \
--arg sha "${MM_SHA}" \
--arg run_id "${GITHUB_RUN_ID}" \
--arg name "${NAME}" \
--arg attempt "${GITHUB_RUN_ATTEMPT}" \
--arg branch "${MM_BRANCH}" \
--arg pr "${PR_NUMBER}" \
'{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}')
else
echo "status_check_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" >> $GITHUB_OUTPUT
COMPOSITE_IDENTITY=$(jq -nc \
--arg repo "${GITHUB_REPOSITORY}" \
--arg sha "${MM_SHA}" \
--arg run_id "${GITHUB_RUN_ID}" \
--arg name "${NAME}" \
--arg attempt "${GITHUB_RUN_ATTEMPT}" \
--arg branch "${MM_BRANCH}" \
'{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}')
fi
echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> $GITHUB_OUTPUT
- name: ci/matrix
id: matrix
env:
INPUT_WORKERS: ${{ inputs.workers }}
run: |
echo "workers=$(jq -nc --argjson n "${INPUT_WORKERS}" '[range(1; $n+1)]')" >> $GITHUB_OUTPUT
echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
run-tests:
# Install webapp node_modules once via the shared webapp-setup action, then
# workers restore the same stable cache. The node_modules cache is keyed only
# on webapp/package-lock.json and is shared with webapp-ci.yml jobs.
prep-deps:
name: prep-deps
runs-on: ubuntu-24.04
timeout-minutes: 10
permissions:
contents: read
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/setup-webapp-node-modules
uses: ./.github/actions/webapp-setup
- name: ci/cache-cypress-deps
# node_modules + the cypress binary (downloaded to ~/.cache/Cypress by
# cypress's postinstall, not into node_modules). Both must be cached;
# otherwise workers see "cypress npm package installed but binary missing".
id: cache-cypress
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: |
e2e-tests/cypress/node_modules
~/.cache/Cypress
key: e2e-cypress-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/cypress/package-lock.json') }}
- name: ci/install-cypress-deps
if: steps.cache-cypress.outputs.cache-hit != 'true'
working-directory: e2e-tests/cypress
run: npm ci
# Register the Test System IO run AFTER prep-deps so workers reach
# dispatch-run within Test System IO's inactivity window.
dispatch-begin:
runs-on: ubuntu-24.04
needs: [prepare-run, prep-deps]
permissions:
contents: read
id-token: write
statuses: write
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/dispatch-begin
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-begin@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
framework: cypress
repo-dir: ${{ github.workspace }}
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
total-reports-expected: ${{ inputs.workers }}
retest-on-fail: ${{ inputs.retest_on_fail }}
cypress-stage: ${{ inputs.cypress_stage }}
cypress-include-group: ${{ inputs.cypress_include_group }}
cypress-exclude-group: ${{ inputs.cypress_exclude_group }}
cypress-skip-on: ${{ inputs.cypress_skip_on }}
cypress-sort-first: ${{ inputs.cypress_sort_first }}
cypress-sort-last: ${{ inputs.cypress_sort_last }}
github-token: ${{ secrets.GITHUB_TOKEN }}
commit-status-context: ${{ inputs.context_name }}
image-tag: ${{ inputs.server_image_tag }}
image-aliases: ${{ inputs.server_image_aliases }}
workers:
name: dispatch-run-${{ matrix.worker_index }}
runs-on: ubuntu-24.04
timeout-minutes: 30
continue-on-error: ${{ inputs.workers > 1 }}
needs:
- generate-test-cycle
if: needs.generate-test-cycle.result == 'success'
needs: [prepare-run, dispatch-begin]
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
matrix:
worker_index: ${{ fromJSON(needs.generate-test-cycle.outputs.workers) }}
defaults:
run:
working-directory: e2e-tests
worker_index: ${{ fromJSON(needs.prepare-run.outputs.workers-matrix) }}
env:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
COMPOSITE_IDENTITY: ${{ needs.prepare-run.outputs.composite-identity-json }}
SERVER: "${{ inputs.server }}"
MM_LICENSE: "${{ secrets.MM_LICENSE }}"
ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}"
TEST: cypress
TEST_FILTER: "${{ inputs.test_filter }}"
BRANCH: "${{ inputs.branch }}-${{ inputs.test_type }}"
# The dispatch adapter invokes `npx cypress run` directly (no
# cross-env), so pin TZ here — several specs assume UTC.
TZ: Etc/UTC
BRANCH: "${{ inputs.branch }}"
BUILD_ID: "${{ inputs.build_id }}"
CI_BASE_URL: "${{ inputs.test_type }}-test-${{ matrix.worker_index }}"
CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
CWS_URL: "${{ secrets.CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}"
steps:
- name: ci/checkout-actions
# Sparse-checkout just .github/actions from the triggering ref (master)
# so the composite action below is available before the full checkout
# overwrites the workspace with inputs.commit_sha.
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
sparse-checkout: .github/actions
sparse-checkout-cone-mode: true
- name: ci/runner-prep-for-openldap
uses: ./.github/actions/runner-prep-openldap
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
- name: ci/setup-webapp-node-modules
uses: ./.github/actions/webapp-setup
with:
node-version-file: ".nvmrc"
- name: ci/npm-cache-verify
# Heal any partial/dangling entries left in the restored ~/.npm cache
# before running `npm ci`. Avoids the intermittent EEXIST/ENOENT
# failures in npm's cacache writer.
run: npm cache verify
- name: ci/get-webapp-node-modules
working-directory: webapp
run: make node_modules
- name: ci/run-tests
run: |
make cloud-init
make
- name: ci/cloud-teardown
if: always()
run: make cloud-teardown
- name: ci/upload-results
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
read-only: "true"
- name: ci/restore-cypress-deps
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-${{ matrix.worker_index }}
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
retention-days: 5
calculate-results:
runs-on: ubuntu-24.04
needs:
- generate-test-cycle
- run-tests
if: always() && needs.generate-test-cycle.result == 'success'
outputs:
passed: ${{ steps.calculate.outputs.passed }}
failed: ${{ steps.calculate.outputs.failed }}
pending: ${{ steps.calculate.outputs.pending }}
total_specs: ${{ steps.calculate.outputs.total_specs }}
failed_specs: ${{ steps.calculate.outputs.failed_specs }}
failed_specs_count: ${{ steps.calculate.outputs.failed_specs_count }}
failed_tests: ${{ steps.calculate.outputs.failed_tests }}
commit_status_message: ${{ steps.calculate.outputs.commit_status_message }}
total: ${{ steps.calculate.outputs.total }}
pass_rate: ${{ steps.calculate.outputs.pass_rate }}
color: ${{ steps.calculate.outputs.color }}
test_duration: ${{ steps.calculate.outputs.test_duration }}
end_time: ${{ steps.record-end-time.outputs.end_time }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: ci/download-results
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
pattern: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-*
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/calculate
id: calculate
uses: ./.github/actions/calculate-cypress-results
with:
original-results-path: e2e-tests/cypress/results
- name: ci/record-end-time
id: record-end-time
run: echo "end_time=$(date +%s)" >> $GITHUB_OUTPUT
run-failed-tests:
runs-on: ubuntu-24.04
timeout-minutes: 30
needs:
- generate-test-cycle
- run-tests
- calculate-results
if: >-
always() &&
needs.calculate-results.result == 'success' &&
needs.calculate-results.outputs.failed != '0' &&
fromJSON(needs.calculate-results.outputs.failed_specs_count) <= 20
defaults:
run:
e2e-tests/cypress/node_modules
~/.cache/Cypress
key: e2e-cypress-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/cypress/package-lock.json') }}
fail-on-cache-miss: true
- name: ci/cloud-init
working-directory: e2e-tests
env:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
SERVER: "${{ inputs.server }}"
MM_LICENSE: "${{ secrets.MM_LICENSE }}"
ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}"
TEST: cypress
BRANCH: "${{ inputs.branch }}-${{ inputs.test_type }}-retest"
BUILD_ID: "${{ inputs.build_id }}-retest"
CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
CWS_URL: "${{ secrets.CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.CWS_EXTRA_HTTP_HEADERS }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
run: make cloud-init
- name: ci/start-server
working-directory: e2e-tests
run: make start-server
- name: ci/dispatch-run
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-run@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version-file: ".nvmrc"
- name: ci/run-failed-specs
env:
SPEC_FILES: ${{ needs.calculate-results.outputs.failed_specs }}
run: |
echo "Retesting failed specs: $SPEC_FILES"
make cloud-init
make start-server run-specs
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
framework: cypress
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
repo-dir: ${{ github.workspace }}
artifacts-root: ${{ github.workspace }}/worker-artifacts
github-token: ${{ secrets.GITHUB_TOKEN }}
gh-job-name: dispatch-run-${{ matrix.worker_index }}
- name: ci/cloud-teardown
if: always()
working-directory: e2e-tests
run: make cloud-teardown
- name: ci/upload-retest-results
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
- name: ci/upload-debug-artifacts
if: always()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-retest-results
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-debug-${{ matrix.worker_index }}
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
e2e-tests/cypress/tests/screenshots/
worker-artifacts/
retention-days: 5
if-no-files-found: ignore
report:
runs-on: ubuntu-24.04
needs:
- generate-test-cycle
- run-tests
- calculate-results
- run-failed-tests
if: always() && needs.calculate-results.result == 'success'
needs: [prepare-run, dispatch-begin, workers]
if: always()
permissions:
contents: read
id-token: write
statuses: write
outputs:
passed: "${{ steps.final-results.outputs.passed }}"
failed: "${{ steps.final-results.outputs.failed }}"
commit_status_message: "${{ steps.final-results.outputs.commit_status_message }}"
duration: "${{ steps.duration.outputs.duration }}"
duration_display: "${{ steps.duration.outputs.duration_display }}"
retest_display: "${{ steps.duration.outputs.retest_display }}"
defaults:
run:
working-directory: e2e-tests
commit_status_description: ${{ steps.summary.outputs.commit_status_description }}
webhook_payload: ${{ steps.summary.outputs.webhook_payload }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
- name: ci/run-summary
id: summary
continue-on-error: true
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
node-version-file: ".nvmrc"
# PATH A: run-failed-tests was skipped (no failures to retest)
- name: ci/download-results-path-a
if: needs.run-failed-tests.result == 'skipped'
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
pattern: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-*
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/use-previous-calculation
if: needs.run-failed-tests.result == 'skipped'
id: use-previous
env:
CALC_PASSED: ${{ needs.calculate-results.outputs.passed }}
CALC_FAILED: ${{ needs.calculate-results.outputs.failed }}
CALC_PENDING: ${{ needs.calculate-results.outputs.pending }}
CALC_TOTAL_SPECS: ${{ needs.calculate-results.outputs.total_specs }}
CALC_FAILED_SPECS: ${{ needs.calculate-results.outputs.failed_specs }}
CALC_FAILED_SPECS_COUNT: ${{ needs.calculate-results.outputs.failed_specs_count }}
CALC_COMMIT_STATUS_MESSAGE: ${{ needs.calculate-results.outputs.commit_status_message }}
CALC_TOTAL: ${{ needs.calculate-results.outputs.total }}
CALC_PASS_RATE: ${{ needs.calculate-results.outputs.pass_rate }}
CALC_COLOR: ${{ needs.calculate-results.outputs.color }}
CALC_TEST_DURATION: ${{ needs.calculate-results.outputs.test_duration }}
CALC_FAILED_TESTS: ${{ needs.calculate-results.outputs.failed_tests }}
run: |
echo "passed=${CALC_PASSED}" >> $GITHUB_OUTPUT
echo "failed=${CALC_FAILED}" >> $GITHUB_OUTPUT
echo "pending=${CALC_PENDING}" >> $GITHUB_OUTPUT
echo "total_specs=${CALC_TOTAL_SPECS}" >> $GITHUB_OUTPUT
echo "failed_specs=${CALC_FAILED_SPECS}" >> $GITHUB_OUTPUT
echo "failed_specs_count=${CALC_FAILED_SPECS_COUNT}" >> $GITHUB_OUTPUT
echo "commit_status_message=${CALC_COMMIT_STATUS_MESSAGE}" >> $GITHUB_OUTPUT
echo "total=${CALC_TOTAL}" >> $GITHUB_OUTPUT
echo "pass_rate=${CALC_PASS_RATE}" >> $GITHUB_OUTPUT
echo "color=${CALC_COLOR}" >> $GITHUB_OUTPUT
echo "test_duration=${CALC_TEST_DURATION}" >> $GITHUB_OUTPUT
{
echo "failed_tests<<EOF"
echo "${CALC_FAILED_TESTS}"
echo "EOF"
} >> $GITHUB_OUTPUT
# PATH B: run-failed-tests ran, need to merge and recalculate
- name: ci/download-original-results
if: needs.run-failed-tests.result != 'skipped'
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
pattern: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-*
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/download-retest-results
if: needs.run-failed-tests.result != 'skipped'
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-retest-results
path: e2e-tests/cypress/retest-results/
- name: ci/calculate-results
if: needs.run-failed-tests.result != 'skipped'
id: recalculate
uses: ./.github/actions/calculate-cypress-results
with:
original-results-path: e2e-tests/cypress/results
retest-results-path: e2e-tests/cypress/retest-results/results
# Set final outputs from either path
- name: ci/set-final-results
id: final-results
env:
RUN_FAILED_TESTS_RESULT: ${{ needs.run-failed-tests.result }}
USE_PREVIOUS_FAILED_TESTS: ${{ steps.use-previous.outputs.failed_tests }}
USE_PREVIOUS_PASSED: ${{ steps.use-previous.outputs.passed }}
USE_PREVIOUS_FAILED: ${{ steps.use-previous.outputs.failed }}
USE_PREVIOUS_PENDING: ${{ steps.use-previous.outputs.pending }}
USE_PREVIOUS_TOTAL_SPECS: ${{ steps.use-previous.outputs.total_specs }}
USE_PREVIOUS_FAILED_SPECS: ${{ steps.use-previous.outputs.failed_specs }}
USE_PREVIOUS_FAILED_SPECS_COUNT: ${{ steps.use-previous.outputs.failed_specs_count }}
USE_PREVIOUS_COMMIT_STATUS_MESSAGE: ${{ steps.use-previous.outputs.commit_status_message }}
USE_PREVIOUS_TOTAL: ${{ steps.use-previous.outputs.total }}
USE_PREVIOUS_PASS_RATE: ${{ steps.use-previous.outputs.pass_rate }}
USE_PREVIOUS_COLOR: ${{ steps.use-previous.outputs.color }}
USE_PREVIOUS_TEST_DURATION: ${{ steps.use-previous.outputs.test_duration }}
RECALCULATE_FAILED_TESTS: ${{ steps.recalculate.outputs.failed_tests }}
RECALCULATE_PASSED: ${{ steps.recalculate.outputs.passed }}
RECALCULATE_FAILED: ${{ steps.recalculate.outputs.failed }}
RECALCULATE_PENDING: ${{ steps.recalculate.outputs.pending }}
RECALCULATE_TOTAL_SPECS: ${{ steps.recalculate.outputs.total_specs }}
RECALCULATE_FAILED_SPECS: ${{ steps.recalculate.outputs.failed_specs }}
RECALCULATE_FAILED_SPECS_COUNT: ${{ steps.recalculate.outputs.failed_specs_count }}
RECALCULATE_COMMIT_STATUS_MESSAGE: ${{ steps.recalculate.outputs.commit_status_message }}
RECALCULATE_TOTAL: ${{ steps.recalculate.outputs.total }}
RECALCULATE_PASS_RATE: ${{ steps.recalculate.outputs.pass_rate }}
RECALCULATE_COLOR: ${{ steps.recalculate.outputs.color }}
RECALCULATE_TEST_DURATION: ${{ steps.recalculate.outputs.test_duration }}
run: |
if [ "$RUN_FAILED_TESTS_RESULT" == "skipped" ]; then
echo "passed=${USE_PREVIOUS_PASSED}" >> $GITHUB_OUTPUT
echo "failed=${USE_PREVIOUS_FAILED}" >> $GITHUB_OUTPUT
echo "pending=${USE_PREVIOUS_PENDING}" >> $GITHUB_OUTPUT
echo "total_specs=${USE_PREVIOUS_TOTAL_SPECS}" >> $GITHUB_OUTPUT
echo "failed_specs=${USE_PREVIOUS_FAILED_SPECS}" >> $GITHUB_OUTPUT
echo "failed_specs_count=${USE_PREVIOUS_FAILED_SPECS_COUNT}" >> $GITHUB_OUTPUT
echo "commit_status_message=${USE_PREVIOUS_COMMIT_STATUS_MESSAGE}" >> $GITHUB_OUTPUT
echo "total=${USE_PREVIOUS_TOTAL}" >> $GITHUB_OUTPUT
echo "pass_rate=${USE_PREVIOUS_PASS_RATE}" >> $GITHUB_OUTPUT
echo "color=${USE_PREVIOUS_COLOR}" >> $GITHUB_OUTPUT
echo "test_duration=${USE_PREVIOUS_TEST_DURATION}" >> $GITHUB_OUTPUT
{
echo "failed_tests<<EOF"
echo "$USE_PREVIOUS_FAILED_TESTS"
echo "EOF"
} >> $GITHUB_OUTPUT
else
echo "passed=${RECALCULATE_PASSED}" >> $GITHUB_OUTPUT
echo "failed=${RECALCULATE_FAILED}" >> $GITHUB_OUTPUT
echo "pending=${RECALCULATE_PENDING}" >> $GITHUB_OUTPUT
echo "total_specs=${RECALCULATE_TOTAL_SPECS}" >> $GITHUB_OUTPUT
echo "failed_specs=${RECALCULATE_FAILED_SPECS}" >> $GITHUB_OUTPUT
echo "failed_specs_count=${RECALCULATE_FAILED_SPECS_COUNT}" >> $GITHUB_OUTPUT
echo "commit_status_message=${RECALCULATE_COMMIT_STATUS_MESSAGE}" >> $GITHUB_OUTPUT
echo "total=${RECALCULATE_TOTAL}" >> $GITHUB_OUTPUT
echo "pass_rate=${RECALCULATE_PASS_RATE}" >> $GITHUB_OUTPUT
echo "color=${RECALCULATE_COLOR}" >> $GITHUB_OUTPUT
echo "test_duration=${RECALCULATE_TEST_DURATION}" >> $GITHUB_OUTPUT
{
echo "failed_tests<<EOF"
echo "$RECALCULATE_FAILED_TESTS"
echo "EOF"
} >> $GITHUB_OUTPUT
fi
- name: ci/compute-duration
id: duration
env:
START_TIME: ${{ needs.generate-test-cycle.outputs.start_time }}
FIRST_PASS_END_TIME: ${{ needs.calculate-results.outputs.end_time }}
RETEST_RESULT: ${{ needs.run-failed-tests.result }}
RETEST_SPEC_COUNT: ${{ needs.calculate-results.outputs.failed_specs_count }}
TEST_DURATION: ${{ steps.final-results.outputs.test_duration }}
run: |
NOW=$(date +%s)
ELAPSED=$((NOW - START_TIME))
MINUTES=$((ELAPSED / 60))
SECONDS=$((ELAPSED % 60))
DURATION="${MINUTES}m ${SECONDS}s"
# Compute first-pass and re-run durations
FIRST_PASS_ELAPSED=$((FIRST_PASS_END_TIME - START_TIME))
FP_MIN=$((FIRST_PASS_ELAPSED / 60))
FP_SEC=$((FIRST_PASS_ELAPSED % 60))
FIRST_PASS="${FP_MIN}m ${FP_SEC}s"
if [ "$RETEST_RESULT" != "skipped" ]; then
RERUN_ELAPSED=$((NOW - FIRST_PASS_END_TIME))
RR_MIN=$((RERUN_ELAPSED / 60))
RR_SEC=$((RERUN_ELAPSED % 60))
RUN_BREAKDOWN=" (first-pass: ${FIRST_PASS}, re-run: ${RR_MIN}m ${RR_SEC}s)"
else
RUN_BREAKDOWN=""
fi
# Duration icons: >20m high alert, >15m warning, otherwise clock
if [ "$MINUTES" -ge 20 ]; then
DURATION_DISPLAY=":rotating_light: ${DURATION}${RUN_BREAKDOWN} | test: ${TEST_DURATION}"
elif [ "$MINUTES" -ge 15 ]; then
DURATION_DISPLAY=":warning: ${DURATION}${RUN_BREAKDOWN} | test: ${TEST_DURATION}"
else
DURATION_DISPLAY=":clock3: ${DURATION}${RUN_BREAKDOWN} | test: ${TEST_DURATION}"
fi
# Retest indicator with spec count
if [ "$RETEST_RESULT" != "skipped" ]; then
RETEST_DISPLAY=":repeat: re-run ${RETEST_SPEC_COUNT} spec(s)"
else
RETEST_DISPLAY=""
fi
echo "duration=${DURATION}" >> $GITHUB_OUTPUT
echo "duration_display=${DURATION_DISPLAY}" >> $GITHUB_OUTPUT
echo "retest_display=${RETEST_DISPLAY}" >> $GITHUB_OUTPUT
- name: ci/upload-combined-results
if: inputs.workers > 1
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
- name: ci/publish-report
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
framework: cypress
report-type: ${{ inputs.report_type }}
image-tag: ${{ inputs.server_image_tag }}
image-aliases: ${{ inputs.server_image_aliases }}
server-image: ${{ env.SERVER_IMAGE }}
pr-number: ${{ inputs.pr_number }}
ref-branch: ${{ inputs.ref_branch }}
commit-status-context: ${{ inputs.context_name }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: ci/publish-webhook
if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != ''
env:
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
COMMIT_STATUS_MESSAGE: ${{ steps.final-results.outputs.commit_status_message }}
COLOR: ${{ steps.final-results.outputs.color }}
REPORT_URL: ${{ needs.generate-test-cycle.outputs.status_check_url }}
TEST_TYPE: ${{ inputs.test_type }}
REPORT_TYPE: ${{ inputs.report_type }}
COMMIT_SHA: ${{ inputs.commit_sha }}
REF_BRANCH: ${{ inputs.ref_branch }}
PR_NUMBER: ${{ inputs.pr_number }}
DURATION_DISPLAY: ${{ steps.duration.outputs.duration_display }}
RETEST_DISPLAY: ${{ steps.duration.outputs.retest_display }}
GITHUB_REPOSITORY: ${{ github.repository }}
SERVER_IMAGE: ${{ env.SERVER_IMAGE }}
PAYLOAD: ${{ steps.summary.outputs.webhook_payload }}
run: |
# Capitalize test type
TEST_TYPE_CAP=$(echo "$TEST_TYPE" | sed 's/.*/\u&/')
# Build source line based on report type
COMMIT_SHORT="${COMMIT_SHA::7}"
COMMIT_URL="https://github.com/${GITHUB_REPOSITORY}/commit/${COMMIT_SHA}"
if [ "$REPORT_TYPE" = "RELEASE_CUT" ]; then
SOURCE_LINE=":github_round: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`"
elif [ "$REPORT_TYPE" = "MASTER" ] || [ "$REPORT_TYPE" = "RELEASE" ]; then
SOURCE_LINE=":git_merge: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`"
else
SOURCE_LINE=":open-pull-request: [mattermost-pr-${PR_NUMBER}](https://github.com/${GITHUB_REPOSITORY}/pull/${PR_NUMBER})"
fi
# Build retest part for message
RETEST_PART=""
if [ -n "$RETEST_DISPLAY" ]; then
RETEST_PART=" | ${RETEST_DISPLAY}"
fi
# Build payload with attachments
PAYLOAD=$(cat <<EOF
{
"username": "E2E Test",
"icon_url": "https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png",
"attachments": [{
"color": "${COLOR}",
"text": "**Results - Cypress ${TEST_TYPE_CAP} Tests**\n\n${SOURCE_LINE}\n:docker: \`${SERVER_IMAGE}\`\n${COMMIT_STATUS_MESSAGE}${RETEST_PART} | [full report](${REPORT_URL})\n${DURATION_DISPLAY}"
}]
}
EOF
)
# Send to webhook
curl -X POST -H "Content-Type: application/json" -d "$PAYLOAD" "$REPORT_WEBHOOK_URL"
- name: ci/write-job-summary
if: always()
env:
STATUS_CHECK_URL: ${{ needs.generate-test-cycle.outputs.status_check_url }}
TEST_TYPE: ${{ inputs.test_type }}
PASSED: ${{ steps.final-results.outputs.passed }}
FAILED: ${{ steps.final-results.outputs.failed }}
PENDING: ${{ steps.final-results.outputs.pending }}
TOTAL_SPECS: ${{ steps.final-results.outputs.total_specs }}
FAILED_SPECS_COUNT: ${{ steps.final-results.outputs.failed_specs_count }}
FAILED_SPECS: ${{ steps.final-results.outputs.failed_specs }}
COMMIT_STATUS_MESSAGE: ${{ steps.final-results.outputs.commit_status_message }}
FAILED_TESTS: ${{ steps.final-results.outputs.failed_tests }}
DURATION_DISPLAY: ${{ steps.duration.outputs.duration_display }}
RETEST_RESULT: ${{ needs.run-failed-tests.result }}
run: |
{
echo "## E2E Test Results - Cypress ${TEST_TYPE}"
echo ""
if [ "$FAILED" = "0" ]; then
echo "All tests passed: **${PASSED} passed**"
else
echo "<details>"
echo "<summary>${FAILED} failed, ${PASSED} passed</summary>"
echo ""
echo "| Test | File |"
echo "|------|------|"
echo "${FAILED_TESTS}"
echo "</details>"
fi
echo ""
echo "### Calculation Outputs"
echo ""
echo "| Output | Value |"
echo "|--------|-------|"
echo "| passed | ${PASSED} |"
echo "| failed | ${FAILED} |"
echo "| pending | ${PENDING} |"
echo "| total_specs | ${TOTAL_SPECS} |"
echo "| failed_specs_count | ${FAILED_SPECS_COUNT} |"
echo "| commit_status_message | ${COMMIT_STATUS_MESSAGE} |"
echo "| failed_specs | ${FAILED_SPECS:-none} |"
echo "| duration | ${DURATION_DISPLAY} |"
if [ "$RETEST_RESULT" != "skipped" ]; then
echo "| retested | Yes |"
else
echo "| retested | No |"
fi
echo ""
echo "---"
echo "[View Full Report](${STATUS_CHECK_URL})"
} >> $GITHUB_STEP_SUMMARY
- name: ci/assert-results
env:
FINAL_RESULTS_FAILED: ${{ steps.final-results.outputs.failed }}
SUMMARY_OUTCOME: ${{ steps.summary.outcome }}
run: |
[ "${FINAL_RESULTS_FAILED}" = "0" ]
update-success-status:
runs-on: ubuntu-24.04
permissions:
contents: read
statuses: write
if: always() && needs.report.result == 'success' && needs.calculate-results.result == 'success'
needs:
- generate-test-cycle
- calculate-results
- report
steps:
- uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "${{ needs.report.outputs.commit_status_message }}, ${{ needs.report.outputs.duration }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: success
target_url: ${{ needs.generate-test-cycle.outputs.status_check_url }}
update-failure-status:
runs-on: ubuntu-24.04
permissions:
contents: read
statuses: write
if: always() && (needs.report.result != 'success' || needs.calculate-results.result != 'success')
needs:
- generate-test-cycle
- calculate-results
- report
steps:
- uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "${{ needs.report.outputs.commit_status_message }}, ${{ needs.report.outputs.duration }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: failure
target_url: ${{ needs.generate-test-cycle.outputs.status_check_url }}
[ "$SUMMARY_OUTCOME" = "success" ]
+3 -43
View File
@@ -176,56 +176,16 @@ jobs:
-f target_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "Posted success for ${CONTEXT_NAME}"
# ── Routing fork ─────────────────────────────────────────────────────
# vars.E2E_USE_TEST_IO_DISPATCH selects between v1 (legacy) and v2.
# vars.E2E_USE_STAGING_TEST_IO_URL toggles v2's staging vs production
# endpoint (default: staging).
cypress-full-v1:
cypress-full:
needs:
- generate-build-variables
if: inputs.should_run != 'false' && vars.E2E_USE_TEST_IO_DISPATCH != 'true'
permissions:
contents: read
statuses: write
uses: ./.github/workflows/e2e-tests-cypress-template.yml
with:
test_type: full
test_filter: '--stage="@prod" --excludeGroup="@te_only,@cloud_only,@high_availability" --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap" --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"'
workers: 40
enabled_docker_services: "postgres inbucket minio openldap elasticsearch keycloak"
commit_sha: ${{ inputs.commit_sha }}
branch: ${{ needs.generate-build-variables.outputs.branch }}
build_id: ${{ needs.generate-build-variables.outputs.build_id }}
server_image_tag: ${{ needs.generate-build-variables.outputs.server_image_tag }}
server_edition: ${{ inputs.server_edition }}
server_image_repo: ${{ inputs.server_image_repo }}
server_image_aliases: ${{ inputs.server_image_aliases }}
server: ${{ inputs.server }}
enable_reporting: ${{ inputs.enable_reporting }}
report_type: ${{ inputs.report_type }}
ref_branch: ${{ inputs.ref_branch }}
pr_number: ${{ inputs.pr_number }}
context_name: "e2e-test/cypress-full/${{ inputs.server_edition || 'enterprise' }}${{ needs.generate-build-variables.outputs.context_suffix }}"
secrets:
MM_LICENSE: ${{ secrets.MM_LICENSE }}
AUTOMATION_DASHBOARD_URL: ${{ secrets.AUTOMATION_DASHBOARD_URL }}
AUTOMATION_DASHBOARD_TOKEN: ${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}
PUSH_NOTIFICATION_SERVER: ${{ secrets.PUSH_NOTIFICATION_SERVER }}
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
CWS_URL: ${{ secrets.CWS_URL }}
CWS_EXTRA_HTTP_HEADERS: ${{ secrets.CWS_EXTRA_HTTP_HEADERS }}
cypress-full-v2:
needs:
- generate-build-variables
if: inputs.should_run != 'false' && vars.E2E_USE_TEST_IO_DISPATCH == 'true'
if: inputs.should_run != 'false'
permissions:
contents: read
statuses: write
id-token: write
pull-requests: write
uses: ./.github/workflows/e2e-tests-cypress-template-v2.yml
uses: ./.github/workflows/e2e-tests-cypress-template.yml
with:
test_type: full
workers: 40
-4
View File
@@ -110,8 +110,6 @@ jobs:
ref_branch: ${{ needs.generate-build-variables.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
# Enterprise FIPS Edition
@@ -161,6 +159,4 @@ jobs:
ref_branch: ${{ needs.generate-build-variables.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
@@ -109,8 +109,6 @@ jobs:
ref_branch: ${{ needs.validate.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
# Enterprise FIPS Edition
@@ -164,6 +162,4 @@ jobs:
ref_branch: ${{ needs.validate.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
@@ -1,390 +0,0 @@
---
name: E2E Tests - Playwright Template (v2 - test system io dispatch)
# Delegates Playwright spec dispatch + reporting to test system io.
# Authenticates via GitHub Actions OIDC; calling job MUST grant
# `id-token: write`.
on:
workflow_call:
inputs:
workers:
description: "Number of parallel test system io dispatch workers"
type: number
required: false
default: 8
enabled_docker_services:
description: "Space-separated list of docker services to enable"
type: string
required: false
default: "postgres inbucket"
commit_sha:
type: string
required: true
branch:
type: string
required: true
build_id:
type: string
required: true
server_image_tag:
description: "Server image tag (e.g., master or short SHA)"
type: string
required: true
server:
type: string
required: false
default: onprem
server_edition:
description: "Server edition: enterprise (default), fips, or team"
type: string
required: false
default: enterprise
server_image_repo:
description: "Docker registry: mattermostdevelopment (default) or mattermost"
type: string
required: false
default: mattermostdevelopment
server_image_aliases:
description: "Comma-separated alias tags for description"
type: string
required: false
enable_reporting:
type: boolean
required: false
default: false
report_type:
type: string
required: false
ref_branch:
type: string
required: false
pr_number:
type: string
required: false
context_name:
description: "GitHub commit status context name"
type: string
required: true
playwright_project:
description: "Playwright project name (passed to dispatch-begin metadata and dispatch-run --project=)."
type: string
required: false
default: chrome
playwright_retries:
description: "Playwright --retries=N (per-spec, in-process retry of flaky tests)"
type: number
required: false
default: 1
retest_on_fail:
description: "Re-dispatch failed dispatch units once (whole-spec retry, on top of Playwright --retries)"
type: boolean
required: false
default: true
secrets:
MM_LICENSE:
required: false
REPORT_WEBHOOK_URL:
required: false
# Callers must grant: contents: read, statuses: write, id-token: write
permissions:
contents: read
statuses: write
id-token: write
env:
SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}"
jobs:
prepare-run:
runs-on: ubuntu-24.04
permissions:
contents: read
id-token: write
statuses: write
outputs:
composite-identity-json: ${{ steps.composite-identity.outputs.composite-identity-json }}
workers-matrix: ${{ steps.matrix.outputs.workers }}
start_time: ${{ steps.matrix.outputs.start_time }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/composite-identity
id: composite-identity
env:
CONTEXT_NAME: ${{ inputs.context_name }}
GITHUB_REPOSITORY: ${{ github.repository }}
MM_BRANCH: ${{ inputs.branch }}
MM_SHA: ${{ inputs.commit_sha }}
PR_NUMBER: ${{ inputs.pr_number }}
run: |
# Derive the test-system-io run name from the GitHub commit-status
# context: drop the `e2e-test/` prefix (the framework name already
# implies E2E in the dashboard) and swap remaining `/` for `-` so
# the dashboard URL is path-safe. The commit-status context itself
# stays unchanged elsewhere — branch protection rules depend on it.
NAME="${CONTEXT_NAME#e2e-test/}"
NAME="${NAME//\//-}"
if [ -n "$PR_NUMBER" ]; then
COMPOSITE_IDENTITY=$(jq -nc \
--arg repo "${GITHUB_REPOSITORY}" \
--arg sha "${MM_SHA}" \
--arg run_id "${GITHUB_RUN_ID}" \
--arg name "${NAME}" \
--arg attempt "${GITHUB_RUN_ATTEMPT}" \
--arg branch "${MM_BRANCH}" \
--arg pr "${PR_NUMBER}" \
'{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}')
else
COMPOSITE_IDENTITY=$(jq -nc \
--arg repo "${GITHUB_REPOSITORY}" \
--arg sha "${MM_SHA}" \
--arg run_id "${GITHUB_RUN_ID}" \
--arg name "${NAME}" \
--arg attempt "${GITHUB_RUN_ATTEMPT}" \
--arg branch "${MM_BRANCH}" \
'{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}')
fi
echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> $GITHUB_OUTPUT
- name: ci/matrix
id: matrix
env:
INPUT_WORKERS: ${{ inputs.workers }}
run: |
echo "workers=$(jq -nc --argjson n "${INPUT_WORKERS}" '[range(1; $n+1)]')" >> $GITHUB_OUTPUT
echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
# Install webapp node_modules once via the shared webapp-setup action, then
# workers restore the same stable cache. The node_modules cache is keyed only
# on webapp/package-lock.json and is shared with webapp-ci.yml jobs.
prep-deps:
name: prep-deps
runs-on: ubuntu-24.04
timeout-minutes: 15
permissions:
contents: read
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/setup-webapp-node-modules
uses: ./.github/actions/webapp-setup
- name: ci/cache-playwright-deps
# Caches node_modules + the rolled-up @mattermost/playwright-lib dist
# so workers don't re-run rollup on every job.
id: cache-playwright
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: |
e2e-tests/playwright/node_modules
e2e-tests/playwright/lib/dist
e2e-tests/playwright/lib/node_modules
key: e2e-playwright-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json', 'e2e-tests/playwright/lib/src/**', 'e2e-tests/playwright/lib/package.json', 'e2e-tests/playwright/lib/rollup.config.js', 'e2e-tests/playwright/lib/tsconfig.json') }}
- name: ci/install-playwright-deps
# `npm ci` creates symlinks at node_modules/@mattermost/{client,types}
# → webapp/platform/{client,types}; targets must already be built.
# The postinstall then builds lib/dist via rollup. Skip browser
# download here — chromium is cached separately below.
if: steps.cache-playwright.outputs.cache-hit != 'true'
working-directory: e2e-tests/playwright
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1"
run: npm ci
- name: ci/cache-playwright-browsers
# Cache chromium binary (~150MB) keyed on the playwright lockfile so a
# version bump invalidates. Restored by workers; no docker image needed.
id: cache-pw-browsers
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: ~/.cache/ms-playwright
key: playwright-browsers-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json') }}
- name: ci/install-playwright-chromium
if: steps.cache-pw-browsers.outputs.cache-hit != 'true'
working-directory: e2e-tests/playwright
run: npx playwright install chromium
# Register the Test System IO run AFTER prep-deps so workers reach
# dispatch-run within Test System IO's inactivity window.
dispatch-begin:
runs-on: ubuntu-24.04
needs: [prepare-run, prep-deps]
permissions:
contents: read
id-token: write
statuses: write
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/dispatch-begin
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-begin@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
framework: playwright
repo-dir: ${{ github.workspace }}
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
total-reports-expected: ${{ inputs.workers }}
retest-on-fail: ${{ inputs.retest_on_fail }}
playwright-project: ${{ inputs.playwright_project }}
github-token: ${{ secrets.GITHUB_TOKEN }}
commit-status-context: ${{ inputs.context_name }}
image-tag: ${{ inputs.server_image_tag }}
image-aliases: ${{ inputs.server_image_aliases }}
workers:
name: dispatch-run-${{ matrix.worker_index }}
runs-on: ubuntu-24.04
timeout-minutes: 30
needs: [prepare-run, dispatch-begin]
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
matrix:
worker_index: ${{ fromJSON(needs.prepare-run.outputs.workers-matrix) }}
env:
COMPOSITE_IDENTITY: ${{ needs.prepare-run.outputs.composite-identity-json }}
SERVER: "${{ inputs.server }}"
MM_LICENSE: "${{ secrets.MM_LICENSE }}"
ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}"
TEST: playwright
BRANCH: "${{ inputs.branch }}"
BUILD_ID: "${{ inputs.build_id }}"
CI_BASE_URL: "full-test-${{ matrix.worker_index }}"
steps:
- name: ci/checkout-actions
# Sparse-checkout just .github/actions from the triggering ref (master)
# so the composite action below is available before the full checkout
# overwrites the workspace with inputs.commit_sha.
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
sparse-checkout: .github/actions
sparse-checkout-cone-mode: true
- name: ci/runner-prep-for-openldap
uses: ./.github/actions/runner-prep-openldap
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-webapp-node-modules
uses: ./.github/actions/webapp-setup
with:
read-only: "true"
- name: ci/restore-playwright-deps
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: |
e2e-tests/playwright/node_modules
e2e-tests/playwright/lib/dist
e2e-tests/playwright/lib/node_modules
key: e2e-playwright-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json', 'e2e-tests/playwright/lib/src/**', 'e2e-tests/playwright/lib/package.json', 'e2e-tests/playwright/lib/rollup.config.js', 'e2e-tests/playwright/lib/tsconfig.json') }}
fail-on-cache-miss: true
- name: ci/restore-playwright-browsers
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: ~/.cache/ms-playwright
key: playwright-browsers-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json') }}
fail-on-cache-miss: true
- name: ci/cloud-init
working-directory: e2e-tests
run: make cloud-init
- name: ci/start-server
working-directory: e2e-tests
run: make start-server
# Run the `setup` project so per-spec dispatches can pass --no-deps
# and skip plugin-load + server-deployment checks. node_modules,
# lib/dist, and chromium are all restored from cache.
- name: ci/prepare-playwright
working-directory: e2e-tests/playwright
run: npx playwright test --project=setup
- name: ci/dispatch-run
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-run@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
framework: playwright
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
repo-dir: ${{ github.workspace }}
artifacts-root: ${{ github.workspace }}/worker-artifacts
github-token: ${{ secrets.GITHUB_TOKEN }}
gh-job-name: dispatch-run-${{ matrix.worker_index }}
playwright-retries: ${{ inputs.playwright_retries }}
playwright-project: ${{ inputs.playwright_project }}
- name: ci/cloud-teardown
if: always()
working-directory: e2e-tests
run: make cloud-teardown
- name: ci/upload-debug-artifacts
if: always()
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: playwright-full-${{ inputs.server_edition }}-debug-${{ matrix.worker_index }}
path: |
e2e-tests/playwright/logs/
e2e-tests/playwright/results/
worker-artifacts/
retention-days: 5
if-no-files-found: ignore
report:
runs-on: ubuntu-24.04
needs: [prepare-run, dispatch-begin, workers]
if: always()
permissions:
contents: read
id-token: write
statuses: write
outputs:
commit_status_description: ${{ steps.summary.outputs.commit_status_description }}
webhook_payload: ${{ steps.summary.outputs.webhook_payload }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: ci/run-summary
id: summary
continue-on-error: true
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
framework: playwright
report-type: ${{ inputs.report_type }}
image-tag: ${{ inputs.server_image_tag }}
image-aliases: ${{ inputs.server_image_aliases }}
server-image: ${{ env.SERVER_IMAGE }}
pr-number: ${{ inputs.pr_number }}
ref-branch: ${{ inputs.ref_branch }}
commit-status-context: ${{ inputs.context_name }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: ci/publish-webhook
if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != ''
env:
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
PAYLOAD: ${{ steps.summary.outputs.webhook_payload }}
run: |
curl -X POST -H "Content-Type: application/json" -d "$PAYLOAD" "$REPORT_WEBHOOK_URL"
- name: ci/assert-results
env:
SUMMARY_OUTCOME: ${{ steps.summary.outcome }}
run: |
[ "$SUMMARY_OUTCOME" = "success" ]
@@ -1,29 +1,24 @@
---
name: E2E Tests - Playwright Template
name: E2E Tests - Playwright Template (test system io dispatch)
# Delegates Playwright spec dispatch + reporting to test system io.
# Authenticates via GitHub Actions OIDC; calling job MUST grant
# `id-token: write`.
on:
workflow_call:
inputs:
# Test configuration
test_type:
description: "Type of test run (smoke or full)"
type: string
required: true
test_filter:
description: "Test filter arguments (e.g., --grep @smoke)"
type: string
required: true
workers:
description: "Number of parallel shards"
description: "Number of parallel test system io dispatch workers"
type: number
required: false
default: 2
default: 8
enabled_docker_services:
description: "Space-separated list of docker services to enable"
type: string
required: false
default: "postgres inbucket"
# Common build variables
commit_sha:
type: string
required: true
@@ -52,11 +47,10 @@ on:
required: false
default: mattermostdevelopment
server_image_aliases:
description: "Comma-separated alias tags for description (e.g., 'release-11.4, release-11')"
description: "Comma-separated alias tags for description"
type: string
required: false
# Reporting options
enable_reporting:
type: boolean
required: false
@@ -65,101 +59,214 @@ on:
type: string
required: false
ref_branch:
description: "Source branch name for webhook messages (e.g., 'master' or 'release-11.4')"
type: string
required: false
pr_number:
type: string
required: false
# Commit status configuration
context_name:
description: "GitHub commit status context name"
type: string
required: true
outputs:
passed:
description: "Number of passed tests"
value: ${{ jobs.report.outputs.passed }}
failed:
description: "Number of failed tests"
value: ${{ jobs.report.outputs.failed }}
report_url:
description: "URL to test report on S3"
value: ${{ jobs.report.outputs.report_url }}
playwright_project:
description: "Playwright project name (passed to dispatch-begin metadata and dispatch-run --project=)."
type: string
required: false
default: chrome
playwright_retries:
description: "Playwright --retries=N (per-spec, in-process retry of flaky tests)"
type: number
required: false
default: 1
retest_on_fail:
description: "Re-dispatch failed dispatch units once (whole-spec retry, on top of Playwright --retries)"
type: boolean
required: false
default: true
secrets:
MM_LICENSE:
required: false
REPORT_WEBHOOK_URL:
required: false
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
# Callers must grant at least these scopes on the job that uses this workflow
# Callers must grant: contents: read, statuses: write, id-token: write
permissions:
contents: read
statuses: write
id-token: write
env:
SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}"
jobs:
update-initial-status:
runs-on: ubuntu-24.04
steps:
- name: ci/set-initial-status
uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "tests running, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: pending
generate-test-variables:
prepare-run:
runs-on: ubuntu-24.04
permissions:
contents: read
id-token: write
statuses: write
outputs:
workers: "${{ steps.generate-workers.outputs.workers }}"
start_time: "${{ steps.generate-workers.outputs.start_time }}"
composite-identity-json: ${{ steps.composite-identity.outputs.composite-identity-json }}
workers-matrix: ${{ steps.matrix.outputs.workers }}
start_time: ${{ steps.matrix.outputs.start_time }}
steps:
- name: ci/generate-workers
id: generate-workers
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/composite-identity
id: composite-identity
env:
CONTEXT_NAME: ${{ inputs.context_name }}
GITHUB_REPOSITORY: ${{ github.repository }}
MM_BRANCH: ${{ inputs.branch }}
MM_SHA: ${{ inputs.commit_sha }}
PR_NUMBER: ${{ inputs.pr_number }}
run: |
# Derive the test-system-io run name from the GitHub commit-status
# context: drop the `e2e-test/` prefix (the framework name already
# implies E2E in the dashboard) and swap remaining `/` for `-` so
# the dashboard URL is path-safe. The commit-status context itself
# stays unchanged elsewhere — branch protection rules depend on it.
NAME="${CONTEXT_NAME#e2e-test/}"
NAME="${NAME//\//-}"
if [ -n "$PR_NUMBER" ]; then
COMPOSITE_IDENTITY=$(jq -nc \
--arg repo "${GITHUB_REPOSITORY}" \
--arg sha "${MM_SHA}" \
--arg run_id "${GITHUB_RUN_ID}" \
--arg name "${NAME}" \
--arg attempt "${GITHUB_RUN_ATTEMPT}" \
--arg branch "${MM_BRANCH}" \
--arg pr "${PR_NUMBER}" \
'{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch, gh_pr_number:$pr}')
else
COMPOSITE_IDENTITY=$(jq -nc \
--arg repo "${GITHUB_REPOSITORY}" \
--arg sha "${MM_SHA}" \
--arg run_id "${GITHUB_RUN_ID}" \
--arg name "${NAME}" \
--arg attempt "${GITHUB_RUN_ATTEMPT}" \
--arg branch "${MM_BRANCH}" \
'{repository:$repo, commit_sha:$sha, gh_run_id:$run_id, name:$name, gh_run_attempt:$attempt, branch:$branch}')
fi
echo "composite-identity-json=${COMPOSITE_IDENTITY}" >> $GITHUB_OUTPUT
- name: ci/matrix
id: matrix
env:
INPUT_WORKERS: ${{ inputs.workers }}
run: |
echo "workers=$(jq -nc --argjson n "${INPUT_WORKERS}" '[range(1; $n + 1)]')" >> $GITHUB_OUTPUT
echo "workers=$(jq -nc --argjson n "${INPUT_WORKERS}" '[range(1; $n+1)]')" >> $GITHUB_OUTPUT
echo "start_time=$(date +%s)" >> $GITHUB_OUTPUT
run-tests:
# Install webapp node_modules once via the shared webapp-setup action, then
# workers restore the same stable cache. The node_modules cache is keyed only
# on webapp/package-lock.json and is shared with webapp-ci.yml jobs.
prep-deps:
name: prep-deps
runs-on: ubuntu-24.04
timeout-minutes: 60
continue-on-error: true
needs:
- generate-test-variables
if: needs.generate-test-variables.result == 'success'
timeout-minutes: 15
permissions:
contents: read
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/setup-webapp-node-modules
uses: ./.github/actions/webapp-setup
- name: ci/cache-playwright-deps
# Caches node_modules + the rolled-up @mattermost/playwright-lib dist
# so workers don't re-run rollup on every job.
id: cache-playwright
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: |
e2e-tests/playwright/node_modules
e2e-tests/playwright/lib/dist
e2e-tests/playwright/lib/node_modules
key: e2e-playwright-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json', 'e2e-tests/playwright/lib/src/**', 'e2e-tests/playwright/lib/package.json', 'e2e-tests/playwright/lib/rollup.config.js', 'e2e-tests/playwright/lib/tsconfig.json') }}
- name: ci/install-playwright-deps
# `npm ci` creates symlinks at node_modules/@mattermost/{client,types}
# → webapp/platform/{client,types}; targets must already be built.
# The postinstall then builds lib/dist via rollup. Skip browser
# download here — chromium is cached separately below.
if: steps.cache-playwright.outputs.cache-hit != 'true'
working-directory: e2e-tests/playwright
env:
PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD: "1"
run: npm ci
- name: ci/cache-playwright-browsers
# Cache chromium binary (~150MB) keyed on the playwright lockfile so a
# version bump invalidates. Restored by workers; no docker image needed.
id: cache-pw-browsers
uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: ~/.cache/ms-playwright
key: playwright-browsers-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json') }}
- name: ci/install-playwright-chromium
if: steps.cache-pw-browsers.outputs.cache-hit != 'true'
working-directory: e2e-tests/playwright
run: npx playwright install chromium
# Register the Test System IO run AFTER prep-deps so workers reach
# dispatch-run within Test System IO's inactivity window.
dispatch-begin:
runs-on: ubuntu-24.04
needs: [prepare-run, prep-deps]
permissions:
contents: read
id-token: write
statuses: write
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 1
- name: ci/dispatch-begin
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-begin@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
framework: playwright
repo-dir: ${{ github.workspace }}
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
total-reports-expected: ${{ inputs.workers }}
retest-on-fail: ${{ inputs.retest_on_fail }}
playwright-project: ${{ inputs.playwright_project }}
github-token: ${{ secrets.GITHUB_TOKEN }}
commit-status-context: ${{ inputs.context_name }}
image-tag: ${{ inputs.server_image_tag }}
image-aliases: ${{ inputs.server_image_aliases }}
workers:
name: dispatch-run-${{ matrix.worker_index }}
runs-on: ubuntu-24.04
timeout-minutes: 30
needs: [prepare-run, dispatch-begin]
permissions:
contents: read
id-token: write
strategy:
fail-fast: false
matrix:
worker_index: ${{ fromJSON(needs.generate-test-variables.outputs.workers) }}
defaults:
run:
working-directory: e2e-tests
worker_index: ${{ fromJSON(needs.prepare-run.outputs.workers-matrix) }}
env:
COMPOSITE_IDENTITY: ${{ needs.prepare-run.outputs.composite-identity-json }}
SERVER: "${{ inputs.server }}"
MM_LICENSE: "${{ secrets.MM_LICENSE }}"
ENABLED_DOCKER_SERVICES: "${{ inputs.enabled_docker_services }}"
TEST: playwright
TEST_FILTER: "${{ inputs.test_filter }}"
PW_SHARD: "${{ format('--shard={0}/{1}', matrix.worker_index, inputs.workers) }}"
BRANCH: "${{ inputs.branch }}-${{ inputs.test_type }}"
BRANCH: "${{ inputs.branch }}"
BUILD_ID: "${{ inputs.build_id }}"
CI_BASE_URL: "${{ inputs.test_type }}-test-${{ matrix.worker_index }}"
CI_BASE_URL: "full-test-${{ matrix.worker_index }}"
steps:
- name: ci/checkout-actions
# Sparse-checkout just .github/actions from the triggering ref (master)
@@ -178,427 +285,106 @@ jobs:
persist-credentials: false
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
- name: ci/setup-webapp-node-modules
uses: ./.github/actions/webapp-setup
with:
node-version-file: ".nvmrc"
- name: ci/npm-cache-verify
# Heal any partial/dangling entries left in the restored ~/.npm cache
# before running `npm ci`. Avoids the intermittent EEXIST/ENOENT
# failures in npm's cacache writer.
run: npm cache verify
- name: ci/get-webapp-node-modules
working-directory: webapp
run: make node_modules
- name: ci/restore-playwright-image-cache
# Cache the Playwright Docker image tar by the SHA of the files that pin
# its version. Cache busts automatically when either file is edited to bump
# the version. Avoids repeated MCR pulls which are frequently blocked by
# Microsoft's CDN ("The request is blocked").
id: playwright-image-cache
uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
read-only: "true"
- name: ci/restore-playwright-deps
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: /tmp/playwright-docker-image.tar
key: playwright-docker-image-${{ hashFiles('e2e-tests/.ci/server.generate.sh', '.github/workflows/e2e-tests-playwright-template.yml') }}-${{ runner.os }}
- name: ci/pre-pull-playwright-image
# Load from cache when available; pull from MCR only on cache miss.
# A single pull attempt is enough because the image is saved to the cache
# tar for all future runs — no need for a retry loop.
run: |
set -euo pipefail
IMAGE="mcr.microsoft.com/playwright:v1.61.0-noble"
TAR="/tmp/playwright-docker-image.tar"
if [ -f "${TAR}" ]; then
echo "Loading Playwright image from GitHub Actions cache"
docker load --input "${TAR}"
else
echo "Cache miss — pulling from MCR"
docker pull "${IMAGE}"
echo "Saving image to cache for future runs"
docker save "${IMAGE}" --output "${TAR}"
fi
- name: ci/run-tests
run: |
make cloud-init
make
path: |
e2e-tests/playwright/node_modules
e2e-tests/playwright/lib/dist
e2e-tests/playwright/lib/node_modules
key: e2e-playwright-deps-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json', 'e2e-tests/playwright/lib/src/**', 'e2e-tests/playwright/lib/package.json', 'e2e-tests/playwright/lib/rollup.config.js', 'e2e-tests/playwright/lib/tsconfig.json') }}
fail-on-cache-miss: true
- name: ci/restore-playwright-browsers
uses: actions/cache/restore@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
with:
path: ~/.cache/ms-playwright
key: playwright-browsers-${{ runner.os }}-${{ hashFiles('e2e-tests/playwright/package-lock.json') }}
fail-on-cache-miss: true
- name: ci/cloud-init
working-directory: e2e-tests
run: make cloud-init
- name: ci/start-server
working-directory: e2e-tests
run: make start-server
# Run the `setup` project so per-spec dispatches can pass --no-deps
# and skip plugin-load + server-deployment checks. node_modules,
# lib/dist, and chromium are all restored from cache.
- name: ci/prepare-playwright
working-directory: e2e-tests/playwright
run: npx playwright test --project=setup
- name: ci/dispatch-run
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-dispatch-run@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
framework: playwright
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
repo-dir: ${{ github.workspace }}
artifacts-root: ${{ github.workspace }}/worker-artifacts
github-token: ${{ secrets.GITHUB_TOKEN }}
gh-job-name: dispatch-run-${{ matrix.worker_index }}
playwright-retries: ${{ inputs.playwright_retries }}
playwright-project: ${{ inputs.playwright_project }}
- name: ci/cloud-teardown
if: always()
working-directory: e2e-tests
run: make cloud-teardown
- name: ci/dump-docker-state-on-failure
# Always run a final docker-state capture so failures unrelated to
# openldap startup (e.g. server container later crashes) still produce
# logs we can inspect. The script's own retry loop dumps openldap
# state per-attempt; this step is a backstop covering the whole job.
if: failure()
run: |
set +e
DIAG="e2e-tests/docker-diagnostics/job-failure"
mkdir -p "$DIAG"
docker ps -a >"$DIAG/docker.ps.txt" 2>&1
docker version >"$DIAG/docker.version.txt" 2>&1
docker info >"$DIAG/docker.info.txt" 2>&1
for c in $(docker ps -a --format '{{.Names}}'); do
docker inspect "$c" >"$DIAG/$c.inspect.json" 2>&1
docker logs "$c" >"$DIAG/$c.log" 2>&1
done
uname -a >"$DIAG/host.uname.txt" 2>&1
free -m >"$DIAG/host.free.txt" 2>&1
df -h >"$DIAG/host.df.txt" 2>&1
sudo dmesg | tail -500 >"$DIAG/host.dmesg.tail.txt" 2>&1
sudo dmesg | grep -iE 'apparmor|denied|oom|killed|openldap|slapd' >"$DIAG/host.dmesg.relevant.txt" 2>&1
- name: ci/upload-docker-diagnostics
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
- name: ci/upload-debug-artifacts
if: always()
with:
name: docker-diagnostics-playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-${{ matrix.worker_index }}
path: e2e-tests/docker-diagnostics/
retention-days: 7
if-no-files-found: ignore
- name: ci/upload-results
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
if: always()
with:
name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-${{ matrix.worker_index }}
name: playwright-full-${{ inputs.server_edition }}-debug-${{ matrix.worker_index }}
path: |
e2e-tests/playwright/logs/
e2e-tests/playwright/results/
worker-artifacts/
retention-days: 5
calculate-results:
runs-on: ubuntu-24.04
needs:
- generate-test-variables
- run-tests
if: always() && needs.generate-test-variables.result == 'success'
outputs:
passed: ${{ steps.calculate.outputs.passed }}
failed: ${{ steps.calculate.outputs.failed }}
flaky: ${{ steps.calculate.outputs.flaky }}
skipped: ${{ steps.calculate.outputs.skipped }}
total_specs: ${{ steps.calculate.outputs.total_specs }}
failed_specs: ${{ steps.calculate.outputs.failed_specs }}
failed_specs_count: ${{ steps.calculate.outputs.failed_specs_count }}
failed_tests: ${{ steps.calculate.outputs.failed_tests }}
commit_status_message: ${{ steps.calculate.outputs.commit_status_message }}
total: ${{ steps.calculate.outputs.total }}
pass_rate: ${{ steps.calculate.outputs.pass_rate }}
passing: ${{ steps.calculate.outputs.passing }}
color: ${{ steps.calculate.outputs.color }}
test_duration: ${{ steps.calculate.outputs.test_duration }}
end_time: ${{ steps.record-end-time.outputs.end_time }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
with:
node-version-file: ".nvmrc"
- name: ci/download-shard-results
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
pattern: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-*
path: e2e-tests/playwright/shard-results/
merge-multiple: true
- name: ci/merge-shard-results
working-directory: e2e-tests/playwright
run: |
mkdir -p results/reporter
# Merge blob reports using Playwright merge-reports (per docs)
npm install --no-save @playwright/test
npx playwright merge-reports --config merge.config.mjs ./shard-results/results/blob-report/
- name: ci/calculate
id: calculate
uses: ./.github/actions/calculate-playwright-results
with:
original-results-path: e2e-tests/playwright/results/reporter/results.json
- name: ci/upload-merged-results
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
with:
name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results
path: e2e-tests/playwright/results/
retention-days: 5
- name: ci/record-end-time
id: record-end-time
run: echo "end_time=$(date +%s)" >> $GITHUB_OUTPUT
# NB: retries for failing specs happen INLINE inside each shard's
# `ci/run-tests` step (see e2e-tests/.ci/server.run_playwright.sh).
# That reuses the already-running server+docker stack instead of
# paying ~4-7 min to provision a fresh one here, and it correctly
# handles the chrome + chrome-serial project split. The old
# standalone `run-failed-tests` job was removed because it was
# invoking `--project=chrome` against specs that only exist in
# chrome-serial, causing the retest to run zero tests.
if-no-files-found: ignore
report:
runs-on: ubuntu-24.04
needs:
- generate-test-variables
- run-tests
- calculate-results
if: always() && needs.calculate-results.result == 'success'
needs: [prepare-run, dispatch-begin, workers]
if: always()
permissions:
contents: read
id-token: write
statuses: write
outputs:
passed: "${{ steps.final-results.outputs.passed }}"
failed: "${{ steps.final-results.outputs.failed }}"
commit_status_message: "${{ steps.final-results.outputs.commit_status_message }}"
report_url: "${{ steps.upload-to-s3.outputs.report_url }}"
duration: "${{ steps.duration.outputs.duration }}"
duration_display: "${{ steps.duration.outputs.duration_display }}"
retest_display: "${{ steps.duration.outputs.retest_display }}"
defaults:
run:
working-directory: e2e-tests
commit_status_description: ${{ steps.summary.outputs.commit_status_description }}
webhook_payload: ${{ steps.summary.outputs.webhook_payload }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
persist-credentials: false
- name: ci/restore-npm-cache
uses: ./.github/actions/restore-e2e-npm-cache
- name: ci/setup-node
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
- name: ci/run-summary
id: summary
continue-on-error: true
uses: mattermost/mattermost-test-system-io/.github/actions/test-system-io-summary@a2ea7f005484c28fedf51e16645f6d3bd683fd63 # 2026-05-16
with:
node-version-file: ".nvmrc"
# Download merged results (uploaded by calculate-results). These blob
# reports already include the inline per-shard retry results, so no
# separate retest download/merge is needed here.
- name: ci/download-results
uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
with:
name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results
path: e2e-tests/playwright/results/
# Calculate final results. Tests that failed in the first pass but
# passed on inline retry are reported as `flaky`, not `failed`, so
# no retest-results-path is needed.
- name: ci/calculate-results
id: final-results
uses: ./.github/actions/calculate-playwright-results
with:
original-results-path: e2e-tests/playwright/results/reporter/results.json
- name: ci/aws-configure
uses: aws-actions/configure-aws-credentials@8df5847569e6427dd6c4fb1cf565c83acfa8afa7 # v6.0.0
with:
aws-region: us-east-1
aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
- name: ci/upload-to-s3
id: upload-to-s3
env:
AWS_REGION: us-east-1
AWS_S3_BUCKET: mattermost-cypress-report
PR_NUMBER: "${{ inputs.pr_number }}"
RUN_ID: "${{ github.run_id }}"
COMMIT_SHA: "${{ inputs.commit_sha }}"
TEST_TYPE: "${{ inputs.test_type }}"
run: |
LOCAL_RESULTS_PATH="playwright/results/"
# Use PR number if available, otherwise use commit SHA prefix
if [ -n "$PR_NUMBER" ]; then
S3_PATH="server-pr-${PR_NUMBER}/e2e-reports/playwright-${TEST_TYPE}/${RUN_ID}"
else
S3_PATH="server-commit-${COMMIT_SHA::7}/e2e-reports/playwright-${TEST_TYPE}/${RUN_ID}"
fi
if [[ -d "$LOCAL_RESULTS_PATH" ]]; then
aws s3 sync "$LOCAL_RESULTS_PATH" "s3://${AWS_S3_BUCKET}/${S3_PATH}/results/" \
--acl public-read --cache-control "no-cache"
fi
REPORT_URL="https://${AWS_S3_BUCKET}.s3.amazonaws.com/${S3_PATH}/results/reporter/index.html"
echo "report_url=$REPORT_URL" >> "$GITHUB_OUTPUT"
- name: ci/compute-duration
id: duration
env:
START_TIME: ${{ needs.generate-test-variables.outputs.start_time }}
FLAKY_COUNT: ${{ steps.final-results.outputs.flaky }}
TEST_DURATION: ${{ steps.final-results.outputs.test_duration }}
run: |
NOW=$(date +%s)
ELAPSED=$((NOW - START_TIME))
MINUTES=$((ELAPSED / 60))
SECONDS=$((ELAPSED % 60))
DURATION="${MINUTES}m ${SECONDS}s"
# Duration icons: >20m high alert, >15m warning, otherwise clock.
# Retries now happen inline per-shard, so there's no separate
# first-pass/re-run breakdown — the shard wall-clock already
# includes any retries it needed.
if [ "$MINUTES" -ge 20 ]; then
DURATION_DISPLAY=":rotating_light: ${DURATION} | test: ${TEST_DURATION}"
elif [ "$MINUTES" -ge 15 ]; then
DURATION_DISPLAY=":warning: ${DURATION} | test: ${TEST_DURATION}"
else
DURATION_DISPLAY=":clock3: ${DURATION} | test: ${TEST_DURATION}"
fi
# Flaky indicator: tests that failed first pass but passed on
# inline retry. Signals retries did run.
if [ -n "$FLAKY_COUNT" ] && [ "$FLAKY_COUNT" -gt 0 ] 2>/dev/null; then
RETEST_DISPLAY=":repeat: ${FLAKY_COUNT} flaky"
else
RETEST_DISPLAY=""
fi
echo "duration=${DURATION}" >> $GITHUB_OUTPUT
echo "duration_display=${DURATION_DISPLAY}" >> $GITHUB_OUTPUT
echo "retest_display=${RETEST_DISPLAY}" >> $GITHUB_OUTPUT
- name: ci/publish-report
use-staging: ${{ vars.E2E_USE_STAGING_TEST_IO_URL != 'false' }}
composite-identity: ${{ needs.prepare-run.outputs.composite-identity-json }}
framework: playwright
report-type: ${{ inputs.report_type }}
image-tag: ${{ inputs.server_image_tag }}
image-aliases: ${{ inputs.server_image_aliases }}
server-image: ${{ env.SERVER_IMAGE }}
pr-number: ${{ inputs.pr_number }}
ref-branch: ${{ inputs.ref_branch }}
commit-status-context: ${{ inputs.context_name }}
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: ci/publish-webhook
if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != ''
env:
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
COMMIT_STATUS_MESSAGE: ${{ steps.final-results.outputs.commit_status_message }}
COLOR: ${{ steps.final-results.outputs.color }}
REPORT_URL: ${{ steps.upload-to-s3.outputs.report_url }}
TEST_TYPE: ${{ inputs.test_type }}
REPORT_TYPE: ${{ inputs.report_type }}
COMMIT_SHA: ${{ inputs.commit_sha }}
REF_BRANCH: ${{ inputs.ref_branch }}
PR_NUMBER: ${{ inputs.pr_number }}
DURATION_DISPLAY: ${{ steps.duration.outputs.duration_display }}
RETEST_DISPLAY: ${{ steps.duration.outputs.retest_display }}
GITHUB_REPOSITORY: ${{ github.repository }}
SERVER_IMAGE: ${{ env.SERVER_IMAGE }}
PAYLOAD: ${{ steps.summary.outputs.webhook_payload }}
run: |
# Capitalize test type
TEST_TYPE_CAP=$(echo "$TEST_TYPE" | sed 's/.*/\u&/')
# Build source line based on report type
COMMIT_SHORT="${COMMIT_SHA::7}"
COMMIT_URL="https://github.com/${GITHUB_REPOSITORY}/commit/${COMMIT_SHA}"
if [ "$REPORT_TYPE" = "RELEASE_CUT" ]; then
SOURCE_LINE=":github_round: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`"
elif [ "$REPORT_TYPE" = "MASTER" ] || [ "$REPORT_TYPE" = "RELEASE" ]; then
SOURCE_LINE=":git_merge: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`"
else
SOURCE_LINE=":open-pull-request: [mattermost-pr-${PR_NUMBER}](https://github.com/${GITHUB_REPOSITORY}/pull/${PR_NUMBER})"
fi
# Build retest part for message
RETEST_PART=""
if [ -n "$RETEST_DISPLAY" ]; then
RETEST_PART=" | ${RETEST_DISPLAY}"
fi
# Build payload with attachments
PAYLOAD=$(cat <<EOF
{
"username": "E2E Test",
"icon_url": "https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png",
"attachments": [{
"color": "${COLOR}",
"text": "**Results - Playwright ${TEST_TYPE_CAP} Tests**\n\n${SOURCE_LINE}\n:docker: \`${SERVER_IMAGE}\`\n${COMMIT_STATUS_MESSAGE}${RETEST_PART} | [full report](${REPORT_URL})\n${DURATION_DISPLAY}"
}]
}
EOF
)
# Send to webhook
curl -X POST -H "Content-Type: application/json" -d "$PAYLOAD" "$REPORT_WEBHOOK_URL"
- name: ci/write-job-summary
if: always()
env:
REPORT_URL: ${{ steps.upload-to-s3.outputs.report_url }}
TEST_TYPE: ${{ inputs.test_type }}
PASSED: ${{ steps.final-results.outputs.passed }}
FAILED: ${{ steps.final-results.outputs.failed }}
FLAKY: ${{ steps.final-results.outputs.flaky }}
SKIPPED: ${{ steps.final-results.outputs.skipped }}
TOTAL_SPECS: ${{ steps.final-results.outputs.total_specs }}
FAILED_SPECS_COUNT: ${{ steps.final-results.outputs.failed_specs_count }}
FAILED_SPECS: ${{ steps.final-results.outputs.failed_specs }}
COMMIT_STATUS_MESSAGE: ${{ steps.final-results.outputs.commit_status_message }}
FAILED_TESTS: ${{ steps.final-results.outputs.failed_tests }}
DURATION_DISPLAY: ${{ steps.duration.outputs.duration_display }}
run: |
{
echo "## E2E Test Results - Playwright ${TEST_TYPE}"
echo ""
if [ "$FAILED" = "0" ]; then
echo "All tests passed: **${PASSED} passed**"
else
echo "<details>"
echo "<summary>${FAILED} failed, ${PASSED} passed</summary>"
echo ""
echo "| Test | File |"
echo "|------|------|"
echo "${FAILED_TESTS}"
echo "</details>"
fi
echo ""
echo "### Calculation Outputs"
echo ""
echo "| Output | Value |"
echo "|--------|-------|"
echo "| passed | ${PASSED} |"
echo "| failed | ${FAILED} |"
echo "| flaky | ${FLAKY} |"
echo "| skipped | ${SKIPPED} |"
echo "| total_specs | ${TOTAL_SPECS} |"
echo "| failed_specs_count | ${FAILED_SPECS_COUNT} |"
echo "| commit_status_message | ${COMMIT_STATUS_MESSAGE} |"
echo "| failed_specs | ${FAILED_SPECS:-none} |"
echo "| duration | ${DURATION_DISPLAY} |"
# Flaky > 0 means some tests needed the inline retry to pass.
if [ -n "$FLAKY" ] && [ "$FLAKY" -gt 0 ] 2>/dev/null; then
echo "| retried (flaky) | ${FLAKY} |"
fi
echo ""
echo "---"
echo "[View Full Report](${REPORT_URL})"
} >> $GITHUB_STEP_SUMMARY
- name: ci/assert-results
env:
FINAL_RESULTS_FAILED: ${{ steps.final-results.outputs.failed }}
SUMMARY_OUTCOME: ${{ steps.summary.outcome }}
run: |
[ "${FINAL_RESULTS_FAILED}" = "0" ]
update-success-status:
runs-on: ubuntu-24.04
if: always() && needs.report.result == 'success' && needs.calculate-results.result == 'success'
needs:
- calculate-results
- report
steps:
- uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "${{ needs.report.outputs.commit_status_message }}, ${{ needs.report.outputs.duration }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: success
target_url: ${{ needs.report.outputs.report_url }}
update-failure-status:
runs-on: ubuntu-24.04
if: always() && (needs.report.result != 'success' || needs.calculate-results.result != 'success')
needs:
- calculate-results
- report
steps:
- uses: mattermost/actions/delivery/update-commit-status@f324ac89b05cc3511cb06e60642ac2fb829f0a63
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "${{ needs.report.outputs.commit_status_message }}, ${{ needs.report.outputs.duration }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: failure
target_url: ${{ needs.report.outputs.report_url }}
[ "$SUMMARY_OUTCOME" = "success" ]
+3 -44
View File
@@ -51,10 +51,6 @@ on:
required: false
REPORT_WEBHOOK_URL:
required: false
AWS_ACCESS_KEY_ID:
required: true
AWS_SECRET_ACCESS_KEY:
required: true
permissions: {}
@@ -170,53 +166,16 @@ jobs:
-f target_url="https://github.com/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
echo "Posted success for ${CONTEXT_NAME}"
# ── Routing fork ─────────────────────────────────────────────────────
# vars.E2E_USE_TEST_IO_DISPATCH selects between v1 (legacy) and v2.
# vars.E2E_USE_STAGING_TEST_IO_URL toggles v2's staging vs production
# endpoint (default: staging).
playwright-full-v1:
playwright-full:
needs:
- generate-build-variables
if: inputs.should_run != 'false' && vars.E2E_USE_TEST_IO_DISPATCH != 'true'
permissions:
contents: read
statuses: write
uses: ./.github/workflows/e2e-tests-playwright-template.yml
with:
test_type: full
test_filter: "--grep-invert @visual"
workers: 8
enabled_docker_services: "postgres inbucket minio openldap elasticsearch keycloak"
commit_sha: ${{ inputs.commit_sha }}
branch: ${{ needs.generate-build-variables.outputs.branch }}
build_id: ${{ needs.generate-build-variables.outputs.build_id }}
server_image_tag: ${{ needs.generate-build-variables.outputs.server_image_tag }}
server_edition: ${{ inputs.server_edition }}
server_image_repo: ${{ inputs.server_image_repo }}
server_image_aliases: ${{ inputs.server_image_aliases }}
server: ${{ inputs.server }}
enable_reporting: ${{ inputs.enable_reporting }}
report_type: ${{ inputs.report_type }}
ref_branch: ${{ inputs.ref_branch }}
pr_number: ${{ inputs.pr_number }}
context_name: "e2e-test/playwright-full/${{ inputs.server_edition || 'enterprise' }}${{ needs.generate-build-variables.outputs.context_suffix }}"
secrets:
MM_LICENSE: ${{ secrets.MM_LICENSE }}
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
playwright-full-v2:
needs:
- generate-build-variables
if: inputs.should_run != 'false' && vars.E2E_USE_TEST_IO_DISPATCH == 'true'
if: inputs.should_run != 'false'
permissions:
contents: read
statuses: write
id-token: write
pull-requests: write
uses: ./.github/workflows/e2e-tests-playwright-template-v2.yml
uses: ./.github/workflows/e2e-tests-playwright-template.yml
with:
workers: 15
enabled_docker_services: "postgres inbucket"