diff --git a/.github/workflows/e2e-fulltests-ci.yml b/.github/workflows/e2e-fulltests-ci.yml new file mode 100644 index 0000000000..d48141769e --- /dev/null +++ b/.github/workflows/e2e-fulltests-ci.yml @@ -0,0 +1,83 @@ +--- +name: E2E Tests +on: + # For PRs, this workflow gets triggered from the Argo Events platform. + # Check the following repo for details: https://github.com/mattermost/delivery-platform + workflow_dispatch: + inputs: + ref: + type: string + required: false + REPORT_TYPE: + type: choice + description: The context this report is being generated in + options: + - PR + - RELEASE + - MASTER + - MASTER_UNSTABLE + - CLOUD + - CLOUD_UNSTABLE + - NONE + default: NONE + +jobs: + generate-test-variables: + runs-on: ubuntu-22.04 + outputs: + status_check_context: "${{ steps.generate.outputs.status_check_context }}" + workers_number: "${{ steps.generate.outputs.workers_number }}" + ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}" + TEST_FILTER: "${{ steps.generate.outputs.TEST_FILTER }}" + env: + # We could exclude the @smoke group for PRs, but then we wouldn't have it in the report + TEST_FILTER_PR: >- + --stage="@prod" + --excludeGroup="@te_only,@cloud_only,@high_availability" + --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap" + --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal" + steps: + - name: ci/generate-test-variables + id: generate + shell: bash + run: | + case "${{ inputs.REPORT_TYPE }}" in + NONE | PR) + echo "status_check_context=E2E Tests/test" >> $GITHUB_OUTPUT + echo "workers_number=20" >> $GITHUB_OUTPUT + echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT + echo "TEST_FILTER=$TEST_FILTER_PR" >> $GITHUB_OUTPUT + ;; + *) + # TODO implement other test types, in the future + echo "Fatal: unimplemented test type. Aborting." + exit 1 + esac + + e2e-fulltest: + needs: + - generate-test-variables + uses: ./.github/workflows/e2e-tests-ci-template.yml + strategy: + matrix: + type: + - name: PR + with: + ref: "${{ inputs.ref || github.sha }}" + status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}" + workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}" + testcase_failure_fatal: false + run_preflight_checks: false + enable_reporting: true + ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}" + TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER }}" + REPORT_TYPE: "${{ inputs.REPORT_TYPE }}" + secrets: + MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}" + AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}" + AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}" + PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}" + REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}" + ### These are disabled until release tests are implemented + #REPORT_TM4J_API_KEY: "${{ secrets.MM_E2E_TM4J_API_KEY }}" + #REPORT_TEST_CYCLE_LINK_PREFIX: "${{ secrets.MM_E2E_TEST_CYCLE_LINK_PREFIX }}" diff --git a/.github/workflows/e2e-tests-ci-template.yml b/.github/workflows/e2e-tests-ci-template.yml new file mode 100644 index 0000000000..74cb680a14 --- /dev/null +++ b/.github/workflows/e2e-tests-ci-template.yml @@ -0,0 +1,390 @@ +--- +name: E2E Tests Template +on: + workflow_call: + inputs: + # NB: this does not support using branch names that belong to forks. + # In those cases, you should specify directly the commit SHA that you want to test, or + # some wrapper workflow that does it for you (e.g. the slash command for initiating a PR test) + ref: + type: string + required: true + status_check_context: + type: string + required: true + workers_number: + type: string # Should ideally be a number; see https://github.com/orgs/community/discussions/67182 + required: false + default: "1" + testcase_failure_fatal: + type: boolean + required: false + default: true + # NB: the following toggles will skip individual steps, rather than the whole jobs, + # to let the dependent jobs run even if these are false + run_preflight_checks: + type: boolean + required: false + default: true + enable_reporting: + type: boolean + required: false + default: false + ENABLED_DOCKER_SERVICES: + type: string + required: false + TEST_FILTER: + type: string + required: false + MM_ENV: + type: string + required: false + REPORT_TYPE: + type: string + required: false + secrets: + MM_LICENSE: + required: false + AUTOMATION_DASHBOARD_URL: + required: false + AUTOMATION_DASHBOARD_TOKEN: + required: false + PUSH_NOTIFICATION_SERVER: + required: false + REPORT_WEBHOOK_URL: + required: false + REPORT_TM4J_API_KEY: + required: false + REPORT_TM4J_TEST_CYCLE_LINK_PREFIX: + required: false + +jobs: + resolve-ref: + runs-on: ubuntu-22.04 + outputs: + commit_sha: "${{ steps.resolve-ref.outputs.commit_sha }}" + steps: + - name: ci/checkout-repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + - name: ci/resolve-ref + id: resolve-ref + run: | + echo "commit_sha=$(git rev-parse --verify HEAD)" >> $GITHUB_OUTPUT + + update-initial-status: + runs-on: ubuntu-22.04 + needs: + - resolve-ref + steps: + - uses: mattermost/actions/delivery/update-commit-status@main + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }} + context: ${{ inputs.status_check_context }} + description: E2E tests for mattermost server app + status: pending + + cypress-check: + runs-on: ubuntu-22.04 + needs: + - update-initial-status + defaults: + run: + working-directory: e2e-tests/cypress + steps: + - name: ci/checkout-repo + if: "${{ inputs.run_preflight_checks }}" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + - name: ci/setup-node + if: "${{ inputs.run_preflight_checks }}" + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + id: setup_node + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: "e2e-tests/cypress/package-lock.json" + - name: ci/cypress/npm-install + if: "${{ inputs.run_preflight_checks }}" + run: | + npm ci + - name: ci/cypress/npm-check + if: "${{ inputs.run_preflight_checks }}" + run: | + npm run check + + playwright-check: + runs-on: ubuntu-22.04 + needs: + - update-initial-status + defaults: + run: + working-directory: e2e-tests/playwright + steps: + - name: ci/checkout-repo + if: "${{ inputs.run_preflight_checks }}" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + - name: ci/setup-node + if: "${{ inputs.run_preflight_checks }}" + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + id: setup_node + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: "e2e-tests/playwright/package-lock.json" + - name: ci/get-webapp-node-modules + if: "${{ inputs.run_preflight_checks }}" + working-directory: webapp + # requires build of client and types + run: | + make node_modules + - name: ci/playwright/npm-install + if: "${{ inputs.run_preflight_checks }}" + run: | + npm ci + - name: ci/playwright/npm-check + if: "${{ inputs.run_preflight_checks }}" + run: | + npm run check + + generate-build-variables: + runs-on: ubuntu-22.04 + needs: + - resolve-ref + - update-initial-status + outputs: + workers: "${{ steps.workers.outputs.workers }}" + BRANCH: "${{ steps.branch.outputs.BRANCH }}" + BUILD_ID: "${{ steps.build-id.outputs.BUILD_ID }}" + steps: + - name: ci/checkout-repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + - name: ci/generate-workers + id: workers + env: + WORKERS: ${{ inputs.workers_number }} + run: | + [ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0 + echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT + - name: ci/generate-branch + id: branch + run: | + PR_LIST=$(curl -L -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${{ github.token }}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "${{ github.api_url }}/repos/${{ github.repository }}/commits/${{ needs.resolve-ref.outputs.commit_sha }}/pulls" 2>/dev/null || echo -n "") + PR_NAMES_LIST=$(echo -n "$PR_LIST" | jq -r 'map(.number) | join("-")') + if [ -n "$PR_NAMES_LIST" ]; then + # This commit belongs to one or more PRs. + # Prefer displaying the PR names in the Automation Dashboard, instead of the branch name + BRANCH="server-pr-${PR_NAMES_LIST}" + else + # This commit does not belong to a PR. Use the given input ref as-is + BRANCH="${{ inputs.ref }}" + fi + echo "Generated branch identifier for E2E run: ${BRANCH}" + echo "BRANCH=${BRANCH}" >> "$GITHUB_OUTPUT" + - name: ci/calculate-build-id + id: build-id + run: | + BUILD_ID="$(git rev-parse --short=7 HEAD)-${{ github.run_id }}-${{ github.run_attempt }}-ent" + echo "Generated BUILD_ID for E2E run: ${BUILD_ID}" + echo "BUILD_ID=${BUILD_ID}" >> "$GITHUB_OUTPUT" + + generate-test-cycle: + runs-on: ubuntu-22.04 + needs: + - generate-build-variables + defaults: + run: + shell: bash + working-directory: e2e-tests + steps: + - name: ci/checkout-repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + - name: ci/setup-node + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + id: setup_node + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: "e2e-tests/cypress/package-lock.json" + - name: ci/e2e-test-gencycle + env: + AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}" + AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}" + BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}" + BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}" + TEST_FILTER: "${{ inputs.TEST_FILTER }}" + run: | + make generate-test-cycle + + test: + continue-on-error: true # Individual runner failures shouldn't prevent the completion of an E2E run + strategy: + fail-fast: false # Individual runner failures shouldn't prevent the completion of an E2E run + matrix: + # + # Note that E2E tests should be run only on ubuntu, for QA purposes. + # But it's useful to be able to run and debug the E2E tests for different OSes. + # Notes: + # - For MacOS: works on developer machines, but uses too many resources to be able to run on Github Actions + # - for Windows: cannot currently run on Github Actions, since the runners do not support running linux containers, at the moment + # + #os: [ubuntu-latest-8-cores, windows-2022, macos-12-xl] + os: [ubuntu-latest-8-cores] + worker_index: ${{ fromJSON(needs.generate-build-variables.outputs.workers) }} # https://docs.github.com/en/actions/learn-github-actions/expressions#example-returning-a-json-object + runs-on: "${{ matrix.os }}" + timeout-minutes: 60 + needs: + - cypress-check + - playwright-check + - generate-build-variables + - generate-test-cycle + defaults: + run: + shell: bash + working-directory: e2e-tests + env: + AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}" + AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}" + MM_LICENSE: "${{ secrets.MM_LICENSE }}" + ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}" + TEST_FILTER: "${{ inputs.TEST_FILTER }}" + MM_ENV: "${{ inputs.MM_ENV }}" + BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}" + BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}" + CI_BASE_URL: "http://localhost:8065/?worker_index=${{ matrix.worker_index }}" + CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}" + steps: + - name: ci/checkout-repo + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + - name: ci/setup-macos-docker + if: runner.os == 'macos' + # https://github.com/actions/runner-images/issues/17#issuecomment-1537238473 + run: | + brew install docker docker-compose + colima start + mkdir -p ~/.docker/cli-plugins + ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose + sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + - name: ci/setup-node + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + id: setup_node + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: "e2e-tests/cypress/package-lock.json" + - name: ci/e2e-test + run: | + make + - name: ci/e2e-test-store-results + uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1 + with: + name: e2e-test-results-${{ matrix.os }}-${{ matrix.worker_index }} + path: | + e2e-tests/cypress/logs/ + e2e-tests/cypress/results/ + - name: ci/e2e-test-assert-results + if: ${{ inputs.testcase_failure_fatal }} + run: | + # Assert that the run contained 0 failures + CYPRESS_FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add) + echo "Cypress run completed with $CYPRESS_FAILURES failures" + [ "$CYPRESS_FAILURES" = "0" ] + + publish-report: + runs-on: ubuntu-22.04 + needs: + - generate-build-variables + - test + defaults: + run: + shell: bash + working-directory: e2e-tests + steps: + - name: ci/checkout-repo + if: "${{ inputs.enable_reporting }}" + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + ref: ${{ inputs.ref }} + fetch-depth: 0 + - name: ci/setup-node + if: "${{ inputs.enable_reporting }}" + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + id: setup_node + with: + node-version-file: ".nvmrc" + cache: npm + cache-dependency-path: "e2e-tests/cypress/package-lock.json" + - name: ci/download-artifacts + if: "${{ inputs.enable_reporting }}" + uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2 + with: + path: e2e-tests/cypress/ + merge-multiple: true + - name: ci/publish-report + if: "${{ inputs.enable_reporting }}" + env: + TYPE: "${{ inputs.REPORT_TYPE }}" + WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}" + BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}" + BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}" + MM_ENV: "${{ inputs.MM_ENV }}" + TM4J_API_KEY: "${{ secrets.REPORT_TM4J_API_KEY }}" + TEST_CYCLE_LINK_PREFIX: "${{ secrets.REPORT_TM4J_TEST_CYCLE_LINK_PREFIX }}" + run: | + make publish-report + + update-failure-final-status: + runs-on: ubuntu-22.04 + if: failure() || cancelled() + needs: + - resolve-ref + - publish-report + steps: + - uses: mattermost/actions/delivery/update-commit-status@main + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }} + context: ${{ inputs.status_check_context }} + description: E2E tests for mattermost server app + status: failure + + update-success-final-status: + runs-on: ubuntu-22.04 + if: success() + needs: + - resolve-ref + - publish-report + steps: + - uses: mattermost/actions/delivery/update-commit-status@main + env: + GITHUB_TOKEN: ${{ github.token }} + with: + repository_full_name: ${{ github.repository }} + commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }} + context: ${{ inputs.status_check_context }} + description: E2E tests for mattermost server app + status: success diff --git a/.github/workflows/e2e-tests-ci.yml b/.github/workflows/e2e-tests-ci.yml index 4eedb3b75c..2b9b1f3af6 100644 --- a/.github/workflows/e2e-tests-ci.yml +++ b/.github/workflows/e2e-tests-ci.yml @@ -1,4 +1,5 @@ -name: E2E Tests +--- +name: E2E Smoketests on: # For PRs, this workflow gets triggered from the Argo Events platform. # Check the following repo for details: https://github.com/mattermost/delivery-platform @@ -9,155 +10,8 @@ on: required: false jobs: - update-initial-status: - runs-on: ubuntu-22.04 - steps: - - uses: mattermost/actions/delivery/update-commit-status@main - env: - GITHUB_TOKEN: ${{ github.token }} - with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ inputs.commit_sha || github.sha }} - context: E2E Tests/smoketests - description: E2E tests for mattermost server app - status: pending - - cypress-check: - runs-on: ubuntu-22.04 - needs: - - update-initial-status - defaults: - run: - working-directory: e2e-tests/cypress - steps: - - name: ci/checkout-repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{ inputs.commit_sha || github.sha }} - - name: ci/setup-node - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 - id: setup_node - with: - node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/cypress/package-lock.json" - - name: ci/cypress/npm-install - run: | - npm ci - - name: ci/cypress/npm-check - run: | - npm run check - - playwright-check: - runs-on: ubuntu-22.04 - needs: - - update-initial-status - defaults: - run: - working-directory: e2e-tests/playwright - steps: - - name: ci/checkout-repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{ inputs.commit_sha || github.sha }} - - name: ci/setup-node - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 - id: setup_node - with: - node-version-file: ".nvmrc" - cache: npm - cache-dependency-path: "e2e-tests/playwright/package-lock.json" - - name: ci/get-webapp-node-modules - working-directory: webapp - # requires build of client and types - run: | - make node_modules - - name: ci/playwright/npm-install - run: | - npm ci - - name: ci/playwright/npm-check - run: | - npm run check - - smoketests: - strategy: - matrix: - # - # Note that smoketests should be run only on ubuntu, for QA purposes. - # But it's useful to be able to run and debug the smoketests for different OSes. - # Notes: - # - For MacOS: works on developer machines, but uses too many resources to be able to run on Github Actions - # - for Windows: cannot currently run on Github Actions, since the runners do not support running linux containers, at the moment - # - #os: [ubuntu-latest-8-cores, windows-2022, macos-12-xl] - os: [ubuntu-latest-8-cores] - runs-on: "${{ matrix.os }}" - needs: - - cypress-check - - playwright-check - defaults: - run: - shell: bash - working-directory: e2e-tests - steps: - - name: ci/checkout-repo - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - ref: ${{ inputs.commit_sha || github.sha }} - - name: ci/setup-macos-docker - if: runner.os == 'macos' - # https://github.com/actions/runner-images/issues/17#issuecomment-1537238473 - run: | - brew install docker docker-compose - colima start - mkdir -p ~/.docker/cli-plugins - ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose - sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - - name: ci/e2e-smoketests - run: | - make - - name: ci/e2e-smoketests-store-results - uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0 - with: - name: e2e-smoketests-results-${{ matrix.os }} - path: | - e2e-tests/cypress/logs/ - e2e-tests/cypress/results/ - - name: ci/e2e-smoketests-assert-results - run: | - # Assert that the run contained 0 failures - CYPRESS_FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add) - echo "Cypress run completed with $CYPRESS_FAILURES failures" - [ "$CYPRESS_FAILURES" = "0" ] - - update-failure-final-status: - runs-on: ubuntu-22.04 - if: failure() || cancelled() - needs: - - smoketests - steps: - - uses: mattermost/actions/delivery/update-commit-status@main - env: - GITHUB_TOKEN: ${{ github.token }} - with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ inputs.commit_sha || github.sha }} - context: E2E Tests/smoketests - description: E2E tests for mattermost server app - status: failure - - update-success-final-status: - runs-on: ubuntu-22.04 - if: success() - needs: - - smoketests - steps: - - uses: mattermost/actions/delivery/update-commit-status@main - env: - GITHUB_TOKEN: ${{ github.token }} - with: - repository_full_name: ${{ github.repository }} - commit_sha: ${{ inputs.commit_sha || github.sha }} - context: E2E Tests/smoketests - description: E2E tests for mattermost server app - status: success + e2e-smoketest: + uses: ./.github/workflows/e2e-tests-ci-template.yml + with: + ref: "${{ inputs.commit_sha || github.sha }}" + status_check_context: "E2E Tests/smoketests" \ No newline at end of file diff --git a/e2e-tests/.ci/.e2erc b/e2e-tests/.ci/.e2erc index 907657ceff..e3fb105050 100644 --- a/e2e-tests/.ci/.e2erc +++ b/e2e-tests/.ci/.e2erc @@ -1,37 +1,5 @@ # shellcheck disable=SC2148,SC2155 -# Utility variables -# NB: these assume you `source` them from the directory this file is in -# Set up base docker compose file and export -export MME2E_DC_SERVER="docker compose -p mmserver -f ./server.yml" -export MME2E_DC_DASHBOARD="docker compose -p mmdashboard -f ./dashboard/docker/docker-compose.yml -f ./dashboard.override.yml" -export MME2E_UID=$(id -u) -export MME2E_OSTYPE=$(docker version -f '{{ .Client.Os }}') -export MME2E_ARCHTYPE=$(docker version -f '{{ .Client.Arch }}') -export NODE_VERSION_REQUIRED=$(cat ../../.nvmrc) - -# Default values for optional variables -export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mattermost-enterprise-edition:$(git rev-parse --short=7 HEAD)" -export BROWSER_DEFAULT="chrome" -export SERVER_DEFAULT="self-hosted" -export TEST_DEFAULT="cypress" -export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket" -export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke' -# OS specific defaults overrides -case $MME2E_OSTYPE in - darwin ) - BROWSER_DEFAULT="electron" ;; - * ) -esac - -# Populate the optional variables that are used in the docker-compose file -export SERVER_IMAGE=${SERVER_IMAGE:-$SERVER_IMAGE_DEFAULT} -export BROWSER=${BROWSER:-$BROWSER_DEFAULT} -export SERVER=${SERVER:-$SERVER_DEFAULT} -export TEST=${TEST:-$TEST_DEFAULT} -export ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES:-$ENABLED_DOCKER_SERVICES_DEFAULT} -export TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT} - # Function definitions mme2e_log () { echo "[$(date +%Y-%m-%dT%H:%M:%S%Z)]" "$@"; } mme2e_get_current_shopt_arg () { @@ -98,10 +66,48 @@ mme2e_is_token_in_list() { grep -qE "(^| )$TOKEN( |$)" <<<"$LIST" } -# Utility alias, for interactive shell usage. Can be reversed with 'unalias docker-compose' in your shell -# NB: this won't work in the script +# Call prerequisite utility functions +mme2e_load_env_file + +# Utility variables +# NB: these assume you `source` them from the directory this file is in +# Set up base docker compose file and export +export MME2E_DC_SERVER="docker compose -p mmserver -f ./server.yml" +export MME2E_DC_DASHBOARD="docker compose -p mmdashboard -f ./dashboard/docker/docker-compose.yml -f ./dashboard.override.yml" +export MME2E_UID=$(id -u) +export MME2E_OSTYPE=$(docker version -f '{{ .Client.Os }}') +export MME2E_ARCHTYPE=$(docker version -f '{{ .Client.Arch }}') +export NODE_VERSION_REQUIRED=$(cat ../../.nvmrc) + +# Utility alias, for interactive shell usage. Can be reversed with 'unalias docker-compose-mmserver' in your shell +# NB: this only works in interactive shells alias docker-compose-mmserver='${MME2E_DC_SERVER}' alias docker-compose-mmdashboard='${MME2E_DC_DASHBOARD}' -# Call prerequisite utility functions -mme2e_load_env_file +# Default values for optional variables +export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mattermost-enterprise-edition:$(git rev-parse --short=7 HEAD)" +export BROWSER_DEFAULT="chrome" +export SERVER_DEFAULT="self-hosted" +export TEST_DEFAULT="cypress" +export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket" +export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke' +export BRANCH_DEFAULT=$(git branch --show-current || echo -n "unknown") +export BUILD_ID_DEFAULT=$(date +%s) +# OS specific defaults overrides +case $MME2E_OSTYPE in + darwin ) + BROWSER_DEFAULT="electron" ;; + * ) +esac + +# Populate the optional variables that are used in the docker-compose file +export SERVER_IMAGE=${SERVER_IMAGE:-$SERVER_IMAGE_DEFAULT} +export BROWSER=${BROWSER:-$BROWSER_DEFAULT} +export SERVER=${SERVER:-$SERVER_DEFAULT} +export TEST=${TEST:-$TEST_DEFAULT} +export ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES:-$ENABLED_DOCKER_SERVICES_DEFAULT} +export TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT} +export BRANCH_UNSANITIZED=${BRANCH:-$BRANCH_DEFAULT} +export BRANCH=${BRANCH_UNSANITIZED::50} # The automation dashboard only accepts branch names up to 50 characters +export BUILD_ID_UNSANITIZED=${BUILD_ID:-$BUILD_ID_DEFAULT} +export BUILD_ID=${BUILD_ID_UNSANITIZED::64} # The automation dashboard only accepts build IDs up to 64 characters diff --git a/e2e-tests/.ci/dashboard.generate_test_cycle.sh b/e2e-tests/.ci/dashboard.generate_test_cycle.sh index fea409663b..603f3edeac 100644 --- a/e2e-tests/.ci/dashboard.generate_test_cycle.sh +++ b/e2e-tests/.ci/dashboard.generate_test_cycle.sh @@ -11,7 +11,13 @@ fi set -a . .env.cypress +if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then + mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Skipping test cycle generation." + exit 0 +fi + mme2e_log "Generating the test cycle on the Automation Dashboard" cd ../cypress +npm i # shellcheck disable=SC2086 exec node --trace-warnings generate_test_cycle.js $TEST_FILTER diff --git a/e2e-tests/.ci/report.publish.sh b/e2e-tests/.ci/report.publish.sh new file mode 100755 index 0000000000..05c5472239 --- /dev/null +++ b/e2e-tests/.ci/report.publish.sh @@ -0,0 +1,65 @@ +#!/bin/bash +# SC2034: appears unused. +# https://www.shellcheck.net/wiki/SC2034 +# shellcheck disable=SC2034 + +set -e -u -o pipefail +cd "$(dirname "$0")" +. .e2erc + +# Default required variables, assert that they are set, and document optional variables +: ${FULL_REPORT:=false} # Valid values: true, false +: ${TYPE:=NONE} # Valid values: PR, RELEASE, MASTER, MASTER_UNSTABLE, CLOUD, CLOUD_UNSTABLE, NONE (which is the same as omitting it) +: ${WEBHOOK_URL:-} # Optional. Mattermost webhook to post the report back to +: ${RELEASE_DATE:-} # Optional. If set, its value will be included in the report as the release date of the tested artifact + +# Env vars used during the test. Their values will be included in the report +: ${BRANCH:?} +: ${BUILD_ID:?} +: ${MM_ENV:-} + +# Populate intermediate variables +export BUILD_TAG="${SERVER_IMAGE##*/}" +export MM_DOCKER_IMAGE="${BUILD_TAG%%:*}" # NB: the 'mattermostdevelopment/' prefix is assumed +export MM_DOCKER_TAG="${BUILD_TAG##*:}" +export SERVER_TYPE="${SERVER}" +# NB: assume that BRANCH follows the convention 'server-pr-${PR_NUMBER}'. If multiple PRs match, the last one is used to generate the link +# Only needed if TYPE=PR +export PULL_REQUEST="https://github.com/mattermost/mattermost/pull/${BRANCH##*-}" + +if [ -n "${TM4J_API_KEY:-}" ]; then + export TM4J_ENABLE=true + export JIRA_PROJECT_KEY=MM + export TM4J_ENVIRONMENT_NAME="${TEST}/${BROWSER}/${SERVER}" + case "${SERVER}" in + cloud) + export TM4J_FOLDER_ID="2014474" ;; + *) + export TM4J_FOLDER_ID="2014475" ;; + esac + : ${TEST_CYCLE_LINK_PREFIX:?} + : ${TM4J_CYCLE_KEY:-} + : ${TM4J_CYCLE_NAME:-} + mme2e_log "TMJ4 integration enabled." +fi + +if [ -n "${DIAGNOSTIC_WEBHOOK_URL:-}" ]; then + : ${DIAGNOSTIC_USER_ID:?} + : ${DIAGNOSTIC_TEAM_ID:?} + mme2e_log "Diagnostic report upload enabled." +fi + +if [ -n "${AWS_S3_BUCKET:-}" ]; then + : ${AWS_ACCESS_KEY_ID:?} + : ${AWS_SECRET_ACCESS_KEY:?} + mme2e_log "S3 report upload enabled." +fi + +cd ../cypress/ +if [ ! -d "results/" ]; then + mme2e_log "Error: 'results/' directory does not exist. Aborting report generation." >&2 + exit 1 +fi + +npm i +node save_report.js diff --git a/e2e-tests/.ci/server.generate.sh b/e2e-tests/.ci/server.generate.sh index 427fc058ec..980a2c4057 100755 --- a/e2e-tests/.ci/server.generate.sh +++ b/e2e-tests/.ci/server.generate.sh @@ -61,7 +61,7 @@ services: MM_TEAMSETTINGS_ENABLEOPENSERVER: "true" MM_SQLSETTINGS_DATASOURCE: "postgres://mmuser:mostest@localhost:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes" MM_SQLSETTINGS_DRIVERNAME: "postgres" - MM_EMAILSETTINGS_SMTPSERVER: "inbucket" + MM_EMAILSETTINGS_SMTPSERVER: "localhost" MM_CLUSTERSETTINGS_READONLYCONFIG: "false" MM_SERVICESETTINGS_ENABLEONBOARDINGFLOW: "false" MM_FEATUREFLAGS_ONBOARDINGTOURTIPS: "false" @@ -259,9 +259,7 @@ EOL generate_env_files() { # Generate .env.server mme2e_log "Generating .env.server" - mme2e_generate_envfile_from_var_names >.env.server <<-EOF - MM_LICENSE - EOF + truncate --size=0 .env.server # Setting SERVER-specific variables case "$SERVER" in @@ -282,16 +280,15 @@ generate_env_files() { done # Generating TEST-specific env files - BRANCH_DEFAULT=$(git branch --show-current) - BUILD_ID_DEFAULT=$(date +%s) - export BRANCH=${BRANCH:-$BRANCH_DEFAULT} - export BUILD_ID=${BUILD_ID:-$BUILD_ID_DEFAULT} - export CI_BASE_URL="${CI_BASE_URL:-localhost}" + # Some are defaulted in .e2erc due to being needed to other scripts as well + export CI_BASE_URL="${CI_BASE_URL:-http://localhost:8065}" export REPO=mattermost # Static, but declared here for making generate_test_cycle.js easier to run export HEADLESS=true # Static, but declared here for making generate_test_cycle.js easier to run case "$TEST" in cypress) mme2e_log "Cypress: Generating .env.cypress" + truncate --size=0 .env.cypress + mme2e_generate_envfile_from_var_names >.env.cypress <<-EOF BRANCH BUILD_ID @@ -299,6 +296,7 @@ generate_env_files() { BROWSER HEADLESS REPO + CYPRESS_pushNotificationServer EOF # Adding service-specific cypress variables for SERVICE in $ENABLED_DOCKER_SERVICES; do @@ -327,8 +325,14 @@ generate_env_files() { echo "CYPRESS_serverEdition=E20" >>.env.cypress ;; esac - # If the dashboard is running, load .env.dashboard into .env.cypress - if DC_COMMAND="$MME2E_DC_DASHBOARD" mme2e_wait_service_healthy dashboard 1; then + # Add Automation Dashboard related variables to cypress container + if [ -n "${AUTOMATION_DASHBOARD_URL:-}" ]; then + mme2e_log "Automation dashboard URL is set: loading related variables into the Cypress container" + mme2e_generate_envfile_from_var_names >>.env.cypress <<-EOF + AUTOMATION_DASHBOARD_URL + AUTOMATION_DASHBOARD_TOKEN + EOF + elif DC_COMMAND="$MME2E_DC_DASHBOARD" mme2e_wait_service_healthy dashboard 1; then mme2e_log "Detected a running automation dashboard: loading its access variables into the Cypress container" cat >>.env.cypress <.env.dashboard fi diff --git a/e2e-tests/.ci/server.prepare.sh b/e2e-tests/.ci/server.prepare.sh index c984c46c27..db9c7f18af 100755 --- a/e2e-tests/.ci/server.prepare.sh +++ b/e2e-tests/.ci/server.prepare.sh @@ -3,6 +3,13 @@ set -e -u -o pipefail cd "$(dirname "$0")" . .e2erc +if [ -n "${MM_LICENSE:-}" ]; then + # We prefer uploading the license here, instead of setting the env var for the server + # This is to retain the flexibility of being able to remove it programmatically, if the tests require it + mme2e_log "Uploading license to server" + ${MME2E_DC_SERVER} exec -T -- server mmctl --local license upload-string "$MM_LICENSE" +fi + if [ "$TEST" = "cypress" ]; then mme2e_log "Prepare Cypress: install dependencies" ${MME2E_DC_SERVER} exec -T -u 0 -- cypress bash -c "id $MME2E_UID || useradd -u $MME2E_UID -m nodeci" # Works around the node image's assumption that the app files are owned by user 1000 diff --git a/e2e-tests/Makefile b/e2e-tests/Makefile index 5ac5248c86..07ce2a16d5 100644 --- a/e2e-tests/Makefile +++ b/e2e-tests/Makefile @@ -39,6 +39,10 @@ cloud-init: requirecmd-jq requirecmd-curl cloud-teardown: bash ./.ci/server.cloud_teardown.sh +.PHONY: publish-report +publish-report: requirecmd-node + bash ./.ci/report.publish.sh + .PHONY: fmt-node fmt-shell fmt requirecmd-%: @which "$(*)" >/dev/null || { echo "Error, missing required CLI tool: $(*). Aborting." >&2; exit 1; } diff --git a/e2e-tests/README.md b/e2e-tests/README.md index 2a8f34ce20..36bafac64d 100644 --- a/e2e-tests/README.md +++ b/e2e-tests/README.md @@ -38,6 +38,7 @@ Instructions, detailed: * When running with `SERVER=cloud`, this will automatically create a cloud customer against the specified `CWS_URL` service, and delete that user after the run is complete. * If you want to run the Playwright tests instead of the Cypress ones, you can run `TEST=playwright make` * If you just want to run a local server instance, without any further testing, you can run `TEST=none make` + * If you're using the automation dashboard, you have the option of sharding the E2E test run: you can launch the `make` command in parallel on different machiness (NB: you must use the same `BUILD_ID` and `BRANCH` values that you used for `make generate-test-cycle`) to distribute running the test cases across them. When doing this, you should also set on each machine the `CI_BASE_URL` variable to a value that uniquely identifies the instance where `make` is running. 4. `make stop`: tears down the server (and the dashboard, if running) * `make clean` will also remove any generated environment or docker-compose files, in addition to stopping the containers. @@ -50,6 +51,7 @@ Notes: * If their value is fixed (e.g. a static server configuration), these may be simply added to the `docker_compose_generator.sh` file, to the appropriate container. * If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir, and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability, and simplifies the question of what container has access to which variable considerably. * Exceptions are of course accepted wherever it makes sense (e.g. if you need to group variables based on some common functionality) +- The `publish-report` Make target is meant for internal usage. Usage and variables are documented in the respective scripts. ##### For code changes: * `make fmt-ci` to format and check yaml files and shell scripts. diff --git a/e2e-tests/cypress/tests/integration/channels/plugins/plugin_startup_fail_spec.js b/e2e-tests/cypress/tests/integration/channels/plugins/plugin_startup_fail_spec.js index 2860a43e8d..09919bd861 100644 --- a/e2e-tests/cypress/tests/integration/channels/plugins/plugin_startup_fail_spec.js +++ b/e2e-tests/cypress/tests/integration/channels/plugins/plugin_startup_fail_spec.js @@ -33,8 +33,8 @@ describe('If plugins fail to start, they can be disabled', () => { it('MM-T2391 If plugins fail to start, they can be disabled', () => { const mimeType = 'application/gzip'; - cy.fixture(gitlabPlugin.filename, 'binary'). - then(Cypress.Blob.binaryStringToBlob). + cy.fixture(gitlabPlugin.filename, null). + then(Cypress.Blob.arrayBufferToBlob). then((fileContent) => { cy.get('input[type=file]').attachFile({fileContent, fileName: gitlabPlugin.filename, mimeType}); }); diff --git a/e2e-tests/cypress/utils/report.js b/e2e-tests/cypress/utils/report.js index b7bfde6e74..c30ab98954 100644 --- a/e2e-tests/cypress/utils/report.js +++ b/e2e-tests/cypress/utils/report.js @@ -246,7 +246,7 @@ function generateTitle() { let dockerImageLink = ''; if (MM_DOCKER_IMAGE && MM_DOCKER_TAG) { - dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermost/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`; + dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermostdevelopment/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`; } let releaseDate = '';