diff --git a/.github/workflows/e2e-fulltests-ci.yml b/.github/workflows/e2e-fulltests-ci.yml index d48141769e..278f76ab2a 100644 --- a/.github/workflows/e2e-fulltests-ci.yml +++ b/.github/workflows/e2e-fulltests-ci.yml @@ -6,6 +6,14 @@ on: workflow_dispatch: inputs: ref: + type: string + description: Branch or SHA of the commit to test. Ignored if PR_NUMBER is specified + required: false + PR_NUMBER: + type: string + description: If testing a PR, specify this instead of ref + required: false + MM_ENV: type: string required: false REPORT_TYPE: @@ -22,13 +30,48 @@ on: default: NONE jobs: + resolve-ref: + runs-on: ubuntu-22.04 + defaults: + run: + shell: bash + outputs: + commit_sha: "${{ steps.resolve-ref.outputs.commit_sha }}" + BRANCH: "${{ steps.resolve-ref.outputs.BRANCH }}" + env: + PR_NUMBER: "${{ inputs.PR_NUMBER }}" + steps: + - name: ci/checkout-repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: "${{ inputs.ref || github.sha }}" + fetch-depth: 0 + - name: ci/resolve-ref + id: resolve-ref + run: | + set -e + if [ -n "$PR_NUMBER" ]; then + curl -fsSL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer ${{ github.token }}" "${{ github.api_url }}/repos/${{ github.repository }}/pulls/${PR_NUMBER}" -o pr.json + echo "commit_sha=$(jq -r .head.sha > $GITHUB_OUTPUT + echo "BRANCH=server-pr-${PR_NUMBER}" >> $GITHUB_OUTPUT + #echo "BRANCH=$(jq -r .head.ref > $GITHUB_OUTPUT + else + echo "commit_sha=$(git rev-parse --verify HEAD)" >> $GITHUB_OUTPUT + echo "BRANCH=${{ inputs.ref || github.sha }}" >> $GITHUB_OUTPUT + fi generate-test-variables: runs-on: ubuntu-22.04 + needs: + - resolve-ref + defaults: + run: + shell: bash outputs: status_check_context: "${{ steps.generate.outputs.status_check_context }}" workers_number: "${{ steps.generate.outputs.workers_number }}" ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}" TEST_FILTER: "${{ steps.generate.outputs.TEST_FILTER }}" + BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}" env: # We could exclude the @smoke group for PRs, but then we wouldn't have it in the report TEST_FILTER_PR: >- @@ -39,14 +82,18 @@ jobs: steps: - name: ci/generate-test-variables id: generate - shell: bash run: | + COMMIT_SHA="${{needs.resolve-ref.outputs.commit_sha}}" + # BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition + # Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23 + BUILD_ID_PREFIX="${{ github.run_id }}_${{ github.run_attempt }}-${COMMIT_SHA::7}" case "${{ inputs.REPORT_TYPE }}" in NONE | PR) echo "status_check_context=E2E Tests/test" >> $GITHUB_OUTPUT echo "workers_number=20" >> $GITHUB_OUTPUT echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT echo "TEST_FILTER=$TEST_FILTER_PR" >> $GITHUB_OUTPUT + echo "BUILD_ID=${BUILD_ID_PREFIX}-pr-onprem-ent" >> $GITHUB_OUTPUT ;; *) # TODO implement other test types, in the future @@ -56,6 +103,7 @@ jobs: e2e-fulltest: needs: + - resolve-ref - generate-test-variables uses: ./.github/workflows/e2e-tests-ci-template.yml strategy: @@ -63,7 +111,7 @@ jobs: type: - name: PR with: - ref: "${{ inputs.ref || github.sha }}" + commit_sha: "${{ needs.resolve-ref.outputs.commit_sha }}" status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}" workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}" testcase_failure_fatal: false @@ -71,6 +119,9 @@ jobs: enable_reporting: true ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}" TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER }}" + MM_ENV: "${{ inputs.MM_ENV || '' }}" + BRANCH: "${{ needs.resolve-ref.outputs.BRANCH }}" + BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}" REPORT_TYPE: "${{ inputs.REPORT_TYPE }}" secrets: MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}" diff --git a/.github/workflows/e2e-tests-ci-template.yml b/.github/workflows/e2e-tests-ci-template.yml index 74cb680a14..8d9bf5a2f1 100644 --- a/.github/workflows/e2e-tests-ci-template.yml +++ b/.github/workflows/e2e-tests-ci-template.yml @@ -6,7 +6,7 @@ on: # NB: this does not support using branch names that belong to forks. # In those cases, you should specify directly the commit SHA that you want to test, or # some wrapper workflow that does it for you (e.g. the slash command for initiating a PR test) - ref: + commit_sha: type: string required: true status_check_context: @@ -39,6 +39,12 @@ on: MM_ENV: type: string required: false + BRANCH: + type: string + required: false + BUILD_ID: + type: string + required: false REPORT_TYPE: type: string required: false @@ -59,32 +65,15 @@ on: required: false jobs: - resolve-ref: - runs-on: ubuntu-22.04 - outputs: - commit_sha: "${{ steps.resolve-ref.outputs.commit_sha }}" - steps: - - name: ci/checkout-repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{ inputs.ref }} - fetch-depth: 0 - - name: ci/resolve-ref - id: resolve-ref - run: | - echo "commit_sha=$(git rev-parse --verify HEAD)" >> $GITHUB_OUTPUT - update-initial-status: runs-on: ubuntu-22.04 - needs: - - resolve-ref steps: - uses: mattermost/actions/delivery/update-commit-status@main env: GITHUB_TOKEN: ${{ github.token }} with: repository_full_name: ${{ github.repository }} - commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }} + commit_sha: ${{ inputs.commit_sha }} context: ${{ inputs.status_check_context }} description: E2E tests for mattermost server app status: pending @@ -101,7 +90,7 @@ jobs: if: "${{ inputs.run_preflight_checks }}" uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: ${{ inputs.ref }} + ref: ${{ inputs.commit_sha }} fetch-depth: 0 - name: ci/setup-node if: "${{ inputs.run_preflight_checks }}" @@ -132,7 +121,7 @@ jobs: if: "${{ inputs.run_preflight_checks }}" uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: ${{ inputs.ref }} + ref: ${{ inputs.commit_sha }} fetch-depth: 0 - name: ci/setup-node if: "${{ inputs.run_preflight_checks }}" @@ -160,17 +149,14 @@ jobs: generate-build-variables: runs-on: ubuntu-22.04 needs: - - resolve-ref - update-initial-status outputs: workers: "${{ steps.workers.outputs.workers }}" - BRANCH: "${{ steps.branch.outputs.BRANCH }}" - BUILD_ID: "${{ steps.build-id.outputs.BUILD_ID }}" steps: - name: ci/checkout-repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: ${{ inputs.ref }} + ref: ${{ inputs.commit_sha }} fetch-depth: 0 - name: ci/generate-workers id: workers @@ -179,44 +165,22 @@ jobs: run: | [ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0 echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT - - name: ci/generate-branch - id: branch - run: | - PR_LIST=$(curl -L -H "Accept: application/vnd.github+json" \ - -H "Authorization: Bearer ${{ github.token }}" \ - -H "X-GitHub-Api-Version: 2022-11-28" \ - "${{ github.api_url }}/repos/${{ github.repository }}/commits/${{ needs.resolve-ref.outputs.commit_sha }}/pulls" 2>/dev/null || echo -n "") - PR_NAMES_LIST=$(echo -n "$PR_LIST" | jq -r 'map(.number) | join("-")') - if [ -n "$PR_NAMES_LIST" ]; then - # This commit belongs to one or more PRs. - # Prefer displaying the PR names in the Automation Dashboard, instead of the branch name - BRANCH="server-pr-${PR_NAMES_LIST}" - else - # This commit does not belong to a PR. Use the given input ref as-is - BRANCH="${{ inputs.ref }}" - fi - echo "Generated branch identifier for E2E run: ${BRANCH}" - echo "BRANCH=${BRANCH}" >> "$GITHUB_OUTPUT" - - name: ci/calculate-build-id - id: build-id - run: | - BUILD_ID="$(git rev-parse --short=7 HEAD)-${{ github.run_id }}-${{ github.run_attempt }}-ent" - echo "Generated BUILD_ID for E2E run: ${BUILD_ID}" - echo "BUILD_ID=${BUILD_ID}" >> "$GITHUB_OUTPUT" generate-test-cycle: runs-on: ubuntu-22.04 needs: - - generate-build-variables + - update-initial-status defaults: run: shell: bash working-directory: e2e-tests + outputs: + status_check_url: "${{ steps.e2e-test-gencycle.outputs.status_check_url }}" steps: - name: ci/checkout-repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: ${{ inputs.ref }} + ref: ${{ inputs.commit_sha }} fetch-depth: 0 - name: ci/setup-node uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 @@ -226,14 +190,23 @@ jobs: cache: npm cache-dependency-path: "e2e-tests/cypress/package-lock.json" - name: ci/e2e-test-gencycle + id: e2e-test-gencycle env: AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}" AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}" - BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}" - BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}" + BRANCH: "${{ inputs.BRANCH }}" + BUILD_ID: "${{ inputs.BUILD_ID }}" TEST_FILTER: "${{ inputs.TEST_FILTER }}" run: | - make generate-test-cycle + set -e -o pipefail + make generate-test-cycle | tee generate-test-cycle.out + # Extract cycle's dashboard URL, if present + TEST_CYCLE_ID=$(sed -nE "s/^.*id: '([^']+)'.*$/\1/p" > $GITHUB_OUTPUT + else + echo "status_check_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT + fi test: continue-on-error: true # Individual runner failures shouldn't prevent the completion of an E2E run @@ -268,15 +241,15 @@ jobs: ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}" TEST_FILTER: "${{ inputs.TEST_FILTER }}" MM_ENV: "${{ inputs.MM_ENV }}" - BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}" - BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}" + BRANCH: "${{ inputs.BRANCH }}" + BUILD_ID: "${{ inputs.BUILD_ID }}" CI_BASE_URL: "http://localhost:8065/?worker_index=${{ matrix.worker_index }}" CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}" steps: - name: ci/checkout-repo uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: ${{ inputs.ref }} + ref: ${{ inputs.commit_sha }} fetch-depth: 0 - name: ci/setup-macos-docker if: runner.os == 'macos' @@ -304,30 +277,39 @@ jobs: path: | e2e-tests/cypress/logs/ e2e-tests/cypress/results/ - - name: ci/e2e-test-assert-results - if: ${{ inputs.testcase_failure_fatal }} - run: | - # Assert that the run contained 0 failures - CYPRESS_FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add) - echo "Cypress run completed with $CYPRESS_FAILURES failures" - [ "$CYPRESS_FAILURES" = "0" ] - publish-report: + report: runs-on: ubuntu-22.04 needs: - - generate-build-variables - test defaults: run: shell: bash working-directory: e2e-tests + outputs: + failures: "${{ steps.calculate-failures.outputs.failures }}" steps: - name: ci/checkout-repo - if: "${{ inputs.enable_reporting }}" uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 with: - ref: ${{ inputs.ref }} + ref: ${{ inputs.commit_sha }} fetch-depth: 0 + - name: ci/download-artifacts + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + with: + path: e2e-tests/cypress/ + merge-multiple: true + - name: ci/report-calculate-failures + id: calculate-failures + run: | + FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add) + echo "failures=${FAILURES}" >> $GITHUB_OUTPUT + echo "Cypress run completed with $FAILURES failures" + - name: ci/e2e-test-assert-results + if: "${{ inputs.testcase_failure_fatal }}" + run: | + # Assert that the run contained 0 failures + [ "${{ steps.calculate-failures.outputs.failures }}" = "0" ] - name: ci/setup-node if: "${{ inputs.enable_reporting }}" uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 @@ -336,19 +318,13 @@ jobs: node-version-file: ".nvmrc" cache: npm cache-dependency-path: "e2e-tests/cypress/package-lock.json" - - name: ci/download-artifacts - if: "${{ inputs.enable_reporting }}" - uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 - with: - path: e2e-tests/cypress/ - merge-multiple: true - name: ci/publish-report if: "${{ inputs.enable_reporting }}" env: TYPE: "${{ inputs.REPORT_TYPE }}" WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}" - BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}" - BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}" + BRANCH: "${{ inputs.BRANCH }}" + BUILD_ID: "${{ inputs.BUILD_ID }}" MM_ENV: "${{ inputs.MM_ENV }}" TM4J_API_KEY: "${{ secrets.REPORT_TM4J_API_KEY }}" TEST_CYCLE_LINK_PREFIX: "${{ secrets.REPORT_TM4J_TEST_CYCLE_LINK_PREFIX }}" @@ -359,32 +335,38 @@ jobs: runs-on: ubuntu-22.04 if: failure() || cancelled() needs: - - resolve-ref - - publish-report + - generate-test-cycle + - test + - report steps: - uses: mattermost/actions/delivery/update-commit-status@main env: GITHUB_TOKEN: ${{ github.token }} with: repository_full_name: ${{ github.repository }} - commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }} + commit_sha: ${{ inputs.commit_sha }} context: ${{ inputs.status_check_context }} - description: E2E tests for mattermost server app + description: | + Completed with ${{ needs.report.outputs.failures }} failures status: failure + target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}" update-success-final-status: runs-on: ubuntu-22.04 if: success() needs: - - resolve-ref - - publish-report + - generate-test-cycle + - test + - report steps: - uses: mattermost/actions/delivery/update-commit-status@main env: GITHUB_TOKEN: ${{ github.token }} with: repository_full_name: ${{ github.repository }} - commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }} + commit_sha: ${{ inputs.commit_sha }} context: ${{ inputs.status_check_context }} - description: E2E tests for mattermost server app + description: | + Completed with ${{ needs.report.outputs.failures }} failures status: success + target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}" diff --git a/.github/workflows/e2e-tests-ci.yml b/.github/workflows/e2e-tests-ci.yml index 2b9b1f3af6..023cb1fc40 100644 --- a/.github/workflows/e2e-tests-ci.yml +++ b/.github/workflows/e2e-tests-ci.yml @@ -13,5 +13,5 @@ jobs: e2e-smoketest: uses: ./.github/workflows/e2e-tests-ci-template.yml with: - ref: "${{ inputs.commit_sha || github.sha }}" - status_check_context: "E2E Tests/smoketests" \ No newline at end of file + commit_sha: "${{ inputs.commit_sha || github.sha }}" + status_check_context: "E2E Tests/smoketests" diff --git a/e2e-tests/Makefile b/e2e-tests/Makefile index 07ce2a16d5..a223cb23f1 100644 --- a/e2e-tests/Makefile +++ b/e2e-tests/Makefile @@ -1,7 +1,3 @@ -# Load environment variables from .env file --include .ci/.env -export - SHELL := /bin/bash .PHONY: all run stop clean diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 36bafac64d..d55289a3bd 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -17,7 +17,7 @@ Instructions, tl;dr: create a local branch with your E2E test changes, then open Instructions, detailed: 1. (optional, undefined variables are set to sane defaults) Create the `.ci/env` file, and populate it with the variables you need out of the following list: * `SERVER`: either `self-hosted` (default) or `cloud`. - * `CWS_URL` (mandatory when `SERVER=cloud`, only used in such caase): when spinning up a cloud-like test server that communicates with a test instance of customer web server. + * `CWS_URL` (mandatory when `SERVER=cloud`, only used in such case): when spinning up a cloud-like test server that communicates with a test instance of customer web server. * `TEST`: either `cypress` (default), `playwright`, or `none` (to avoid creating the cypress/playwright sidecar containers, e.g. if you only want to launch a server instance) * `ENABLED_DOCKER_SERVICES`: a space-separated list of services to start alongside the server. Default to `postgres inbucket`, for smoke tests purposes and for lightweight and faster start up time. Depending on the test requirement being worked on, you may want to override as needed, as such: - Cypress full tests require all services to be running: `postgres inbucket minio openldap elasticsearch keycloak`. @@ -29,6 +29,7 @@ Instructions, detailed: * The `TEST_FILTER` variable can also be set, to customize which tests you want cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format. 2. (optional) `make start-dashboard && make generate-test-cycle`: start the automation-dashboard in the background, and initiate a test cycle on it, for the given `BUILD_ID` * NB: the `BUILD_ID` value should stay the same across the `make generate-test-cycle` command, and the subsequent `make` (see next step). If you need to initiate a new test cycle on the same dashboard, you'll need to change the `BUILD_ID` value and rerun both `make generate-test-cycle` and `make`. + * Note that part of the dashboard functionality assumes the `BUILD_ID` to have a certain format (see [here](https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23) for details). This is not relevant for local running, but it's important to note in the testing pipelines. * This also automatically sets the `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` variables for the cypress container * Note that if you run the dashboard locally, but also specify other `AUTOMATION_DASHBOARD_*` variables in your `.ci/env` file, the latter variables will take precedence. * The dashboard is used for orchestrating specs with parallel test run and is typically used in CI. @@ -40,7 +41,7 @@ Instructions, detailed: * If you just want to run a local server instance, without any further testing, you can run `TEST=none make` * If you're using the automation dashboard, you have the option of sharding the E2E test run: you can launch the `make` command in parallel on different machiness (NB: you must use the same `BUILD_ID` and `BRANCH` values that you used for `make generate-test-cycle`) to distribute running the test cases across them. When doing this, you should also set on each machine the `CI_BASE_URL` variable to a value that uniquely identifies the instance where `make` is running. 4. `make stop`: tears down the server (and the dashboard, if running) - * `make clean` will also remove any generated environment or docker-compose files, in addition to stopping the containers. + * This also implicitly runs `make clean`, which also remove any generated environment or docker-compose files. Notes: - Setting a variable in `.ci/env` is functionally equivalent to exporting variables in your current shell's environment, before invoking the makefile.