IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

CLD-7074 Implement full E2E tests on Github Actions (#26093)

Browse Source 
* Move license setting from env var to mmctl upload
* Extract common E2E steps
* Add E2E fulltests
* Fix plugin_startup_fail_spec.js, timeout runners, fix local dashboard run
* Implement reporting script and job
* Bump artifact related actions
* Fix E2E variable generator script
* Skip reporting steps if not required
* Get rid of deprecation warnings
* Fix inbucket hostname parameter
* Support arbitrary refs in test template
* Fix cycle BRANCH var for non-PR commits
---------

Co-authored-by: Mattermost Build <build@mattermost.com>
This commit is contained in:
Mario Vitale 2024-03-06 11:27:53 +01:00 committed by GitHub
parent c82352c90c
commit d7a77d8c42
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 624 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

83
.github/workflows/e2e-fulltests-ci.yml vendored Normal file
View File

@ -0,0 +1,83 @@
---
name: E2E Tests
on:
# For PRs, this workflow gets triggered from the Argo Events platform.
# Check the following repo for details: https://github.com/mattermost/delivery-platform
workflow_dispatch:
inputs:
ref:
type: string
required: false
REPORT_TYPE:
type: choice
description: The context this report is being generated in
options:
- PR
- RELEASE
- MASTER
- MASTER_UNSTABLE
- CLOUD
- CLOUD_UNSTABLE
- NONE
default: NONE
jobs:
generate-test-variables:
runs-on: ubuntu-22.04
outputs:
status_check_context: "${{ steps.generate.outputs.status_check_context }}"
workers_number: "${{ steps.generate.outputs.workers_number }}"
ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER: "${{ steps.generate.outputs.TEST_FILTER }}"
env:
# We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
TEST_FILTER_PR: >-
--stage="@prod"
--excludeGroup="@te_only,@cloud_only,@high_availability"
--sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap"
--sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
steps:
- name: ci/generate-test-variables
id: generate
shell: bash
run: |
case "${{ inputs.REPORT_TYPE }}" in
NONE | PR)
echo "status_check_context=E2E Tests/test" >> $GITHUB_OUTPUT
echo "workers_number=20" >> $GITHUB_OUTPUT
echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT
echo "TEST_FILTER=$TEST_FILTER_PR" >> $GITHUB_OUTPUT
;;
*)
# TODO implement other test types, in the future
echo "Fatal: unimplemented test type. Aborting."
exit 1
esac
e2e-fulltest:
needs:
- generate-test-variables
uses: ./.github/workflows/e2e-tests-ci-template.yml
strategy:
matrix:
type:
- name: PR
with:
ref: "${{ inputs.ref || github.sha }}"
status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}"
workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}"
testcase_failure_fatal: false
run_preflight_checks: false
enable_reporting: true
ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER }}"
REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}"
PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
### These are disabled until release tests are implemented
#REPORT_TM4J_API_KEY: "${{ secrets.MM_E2E_TM4J_API_KEY }}"
#REPORT_TEST_CYCLE_LINK_PREFIX: "${{ secrets.MM_E2E_TEST_CYCLE_LINK_PREFIX }}"

390
.github/workflows/e2e-tests-ci-template.yml vendored Normal file
View File

@ -0,0 +1,390 @@
---
name: E2E Tests Template
on:
workflow_call:
inputs:
# NB: this does not support using branch names that belong to forks.
# In those cases, you should specify directly the commit SHA that you want to test, or
# some wrapper workflow that does it for you (e.g. the slash command for initiating a PR test)
ref:
type: string
required: true
status_check_context:
type: string
required: true
workers_number:
type: string # Should ideally be a number; see https://github.com/orgs/community/discussions/67182
required: false
default: "1"
testcase_failure_fatal:
type: boolean
required: false
default: true
# NB: the following toggles will skip individual steps, rather than the whole jobs,
# to let the dependent jobs run even if these are false
run_preflight_checks:
type: boolean
required: false
default: true
enable_reporting:
type: boolean
required: false
default: false
ENABLED_DOCKER_SERVICES:
type: string
required: false
TEST_FILTER:
type: string
required: false
MM_ENV:
type: string
required: false
REPORT_TYPE:
type: string
required: false
secrets:
MM_LICENSE:
required: false
AUTOMATION_DASHBOARD_URL:
required: false
AUTOMATION_DASHBOARD_TOKEN:
required: false
PUSH_NOTIFICATION_SERVER:
required: false
REPORT_WEBHOOK_URL:
required: false
REPORT_TM4J_API_KEY:
required: false
REPORT_TM4J_TEST_CYCLE_LINK_PREFIX:
required: false
jobs:
resolve-ref:
runs-on: ubuntu-22.04
outputs:
commit_sha: "${{ steps.resolve-ref.outputs.commit_sha }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: ci/resolve-ref
id: resolve-ref
run: |
echo "commit_sha=$(git rev-parse --verify HEAD)" >> $GITHUB_OUTPUT
update-initial-status:
runs-on: ubuntu-22.04
needs:
- resolve-ref
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: E2E tests for mattermost server app
status: pending
cypress-check:
runs-on: ubuntu-22.04
needs:
- update-initial-status
defaults:
run:
working-directory: e2e-tests/cypress
steps:
- name: ci/checkout-repo
if: "${{ inputs.run_preflight_checks }}"
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: ci/setup-node
if: "${{ inputs.run_preflight_checks }}"
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
id: setup_node
with:
node-version-file: ".nvmrc"
cache: npm
cache-dependency-path: "e2e-tests/cypress/package-lock.json"
- name: ci/cypress/npm-install
if: "${{ inputs.run_preflight_checks }}"
run: |
npm ci
- name: ci/cypress/npm-check
if: "${{ inputs.run_preflight_checks }}"
run: |
npm run check
playwright-check:
runs-on: ubuntu-22.04
needs:
- update-initial-status
defaults:
run:
working-directory: e2e-tests/playwright
steps:
- name: ci/checkout-repo
if: "${{ inputs.run_preflight_checks }}"
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: ci/setup-node
if: "${{ inputs.run_preflight_checks }}"
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
id: setup_node
with:
node-version-file: ".nvmrc"
cache: npm
cache-dependency-path: "e2e-tests/playwright/package-lock.json"
- name: ci/get-webapp-node-modules
if: "${{ inputs.run_preflight_checks }}"
working-directory: webapp
# requires build of client and types
run: |
make node_modules
- name: ci/playwright/npm-install
if: "${{ inputs.run_preflight_checks }}"
run: |
npm ci
- name: ci/playwright/npm-check
if: "${{ inputs.run_preflight_checks }}"
run: |
npm run check
generate-build-variables:
runs-on: ubuntu-22.04
needs:
- resolve-ref
- update-initial-status
outputs:
workers: "${{ steps.workers.outputs.workers }}"
BRANCH: "${{ steps.branch.outputs.BRANCH }}"
BUILD_ID: "${{ steps.build-id.outputs.BUILD_ID }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: ci/generate-workers
id: workers
env:
WORKERS: ${{ inputs.workers_number }}
run: |
[ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0
echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT
- name: ci/generate-branch
id: branch
run: |
PR_LIST=$(curl -L -H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ github.token }}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"${{ github.api_url }}/repos/${{ github.repository }}/commits/${{ needs.resolve-ref.outputs.commit_sha }}/pulls" 2>/dev/null || echo -n "")
PR_NAMES_LIST=$(echo -n "$PR_LIST" | jq -r 'map(.number) | join("-")')
if [ -n "$PR_NAMES_LIST" ]; then
# This commit belongs to one or more PRs.
# Prefer displaying the PR names in the Automation Dashboard, instead of the branch name
BRANCH="server-pr-${PR_NAMES_LIST}"
else
# This commit does not belong to a PR. Use the given input ref as-is
BRANCH="${{ inputs.ref }}"
fi
echo "Generated branch identifier for E2E run: ${BRANCH}"
echo "BRANCH=${BRANCH}" >> "$GITHUB_OUTPUT"
- name: ci/calculate-build-id
id: build-id
run: |
BUILD_ID="$(git rev-parse --short=7 HEAD)-${{ github.run_id }}-${{ github.run_attempt }}-ent"
echo "Generated BUILD_ID for E2E run: ${BUILD_ID}"
echo "BUILD_ID=${BUILD_ID}" >> "$GITHUB_OUTPUT"
generate-test-cycle:
runs-on: ubuntu-22.04
needs:
- generate-build-variables
defaults:
run:
shell: bash
working-directory: e2e-tests
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: ci/setup-node
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
id: setup_node
with:
node-version-file: ".nvmrc"
cache: npm
cache-dependency-path: "e2e-tests/cypress/package-lock.json"
- name: ci/e2e-test-gencycle
env:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}"
TEST_FILTER: "${{ inputs.TEST_FILTER }}"
run: |
make generate-test-cycle
test:
continue-on-error: true # Individual runner failures shouldn't prevent the completion of an E2E run
strategy:
fail-fast: false # Individual runner failures shouldn't prevent the completion of an E2E run
matrix:
#
# Note that E2E tests should be run only on ubuntu, for QA purposes.
# But it's useful to be able to run and debug the E2E tests for different OSes.
# Notes:
# - For MacOS: works on developer machines, but uses too many resources to be able to run on Github Actions
# - for Windows: cannot currently run on Github Actions, since the runners do not support running linux containers, at the moment
#
#os: [ubuntu-latest-8-cores, windows-2022, macos-12-xl]
os: [ubuntu-latest-8-cores]
worker_index: ${{ fromJSON(needs.generate-build-variables.outputs.workers) }} # https://docs.github.com/en/actions/learn-github-actions/expressions#example-returning-a-json-object
runs-on: "${{ matrix.os }}"
timeout-minutes: 60
needs:
- cypress-check
- playwright-check
- generate-build-variables
- generate-test-cycle
defaults:
run:
shell: bash
working-directory: e2e-tests
env:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
MM_LICENSE: "${{ secrets.MM_LICENSE }}"
ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER: "${{ inputs.TEST_FILTER }}"
MM_ENV: "${{ inputs.MM_ENV }}"
BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}"
CI_BASE_URL: "http://localhost:8065/?worker_index=${{ matrix.worker_index }}"
CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: ci/setup-macos-docker
if: runner.os == 'macos'
# https://github.com/actions/runner-images/issues/17#issuecomment-1537238473
run: |
brew install docker docker-compose
colima start
mkdir -p ~/.docker/cli-plugins
ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
- name: ci/setup-node
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
id: setup_node
with:
node-version-file: ".nvmrc"
cache: npm
cache-dependency-path: "e2e-tests/cypress/package-lock.json"
- name: ci/e2e-test
run: |
make
- name: ci/e2e-test-store-results
uses: actions/upload-artifact@5d5d22a31266ced268874388b861e4b58bb5c2f3 # v4.3.1
with:
name: e2e-test-results-${{ matrix.os }}-${{ matrix.worker_index }}
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
- name: ci/e2e-test-assert-results
if: ${{ inputs.testcase_failure_fatal }}
run: |
# Assert that the run contained 0 failures
CYPRESS_FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
echo "Cypress run completed with $CYPRESS_FAILURES failures"
[ "$CYPRESS_FAILURES" = "0" ]
publish-report:
runs-on: ubuntu-22.04
needs:
- generate-build-variables
- test
defaults:
run:
shell: bash
working-directory: e2e-tests
steps:
- name: ci/checkout-repo
if: "${{ inputs.enable_reporting }}"
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: ci/setup-node
if: "${{ inputs.enable_reporting }}"
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
id: setup_node
with:
node-version-file: ".nvmrc"
cache: npm
cache-dependency-path: "e2e-tests/cypress/package-lock.json"
- name: ci/download-artifacts
if: "${{ inputs.enable_reporting }}"
uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/publish-report
if: "${{ inputs.enable_reporting }}"
env:
TYPE: "${{ inputs.REPORT_TYPE }}"
WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}"
BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}"
MM_ENV: "${{ inputs.MM_ENV }}"
TM4J_API_KEY: "${{ secrets.REPORT_TM4J_API_KEY }}"
TEST_CYCLE_LINK_PREFIX: "${{ secrets.REPORT_TM4J_TEST_CYCLE_LINK_PREFIX }}"
run: |
make publish-report
update-failure-final-status:
runs-on: ubuntu-22.04
if: failure() || cancelled()
needs:
- resolve-ref
- publish-report
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: E2E tests for mattermost server app
status: failure
update-success-final-status:
runs-on: ubuntu-22.04
if: success()
needs:
- resolve-ref
- publish-report
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: E2E tests for mattermost server app
status: success

160
.github/workflows/e2e-tests-ci.yml vendored
View File

@ -1,4 +1,5 @@
name: E2E Tests
---
name: E2E Smoketests
on:
# For PRs, this workflow gets triggered from the Argo Events platform.
# Check the following repo for details: https://github.com/mattermost/delivery-platform
@ -9,155 +10,8 @@ on:
required: false
jobs:
update-initial-status:
runs-on: ubuntu-22.04
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha || github.sha }}
context: E2E Tests/smoketests
description: E2E tests for mattermost server app
status: pending
cypress-check:
runs-on: ubuntu-22.04
needs:
- update-initial-status
defaults:
run:
working-directory: e2e-tests/cypress
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.commit_sha || github.sha }}
- name: ci/setup-node
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
id: setup_node
with:
node-version-file: ".nvmrc"
cache: npm
cache-dependency-path: "e2e-tests/cypress/package-lock.json"
- name: ci/cypress/npm-install
run: |
npm ci
- name: ci/cypress/npm-check
run: |
npm run check
playwright-check:
runs-on: ubuntu-22.04
needs:
- update-initial-status
defaults:
run:
working-directory: e2e-tests/playwright
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.commit_sha || github.sha }}
- name: ci/setup-node
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
id: setup_node
with:
node-version-file: ".nvmrc"
cache: npm
cache-dependency-path: "e2e-tests/playwright/package-lock.json"
- name: ci/get-webapp-node-modules
working-directory: webapp
# requires build of client and types
run: |
make node_modules
- name: ci/playwright/npm-install
run: |
npm ci
- name: ci/playwright/npm-check
run: |
npm run check
smoketests:
strategy:
matrix:
#
# Note that smoketests should be run only on ubuntu, for QA purposes.
# But it's useful to be able to run and debug the smoketests for different OSes.
# Notes:
# - For MacOS: works on developer machines, but uses too many resources to be able to run on Github Actions
# - for Windows: cannot currently run on Github Actions, since the runners do not support running linux containers, at the moment
#
#os: [ubuntu-latest-8-cores, windows-2022, macos-12-xl]
os: [ubuntu-latest-8-cores]
runs-on: "${{ matrix.os }}"
needs:
- cypress-check
- playwright-check
defaults:
run:
shell: bash
working-directory: e2e-tests
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.commit_sha || github.sha }}
- name: ci/setup-macos-docker
if: runner.os == 'macos'
# https://github.com/actions/runner-images/issues/17#issuecomment-1537238473
run: |
brew install docker docker-compose
colima start
mkdir -p ~/.docker/cli-plugins
ln -sfn /usr/local/opt/docker-compose/bin/docker-compose ~/.docker/cli-plugins/docker-compose
sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
- name: ci/e2e-smoketests
run: |
make
- name: ci/e2e-smoketests-store-results
uses: actions/upload-artifact@26f96dfa697d77e81fd5907df203aa23a56210a8 # v4.3.0
with:
name: e2e-smoketests-results-${{ matrix.os }}
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
- name: ci/e2e-smoketests-assert-results
run: |
# Assert that the run contained 0 failures
CYPRESS_FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
echo "Cypress run completed with $CYPRESS_FAILURES failures"
[ "$CYPRESS_FAILURES" = "0" ]
update-failure-final-status:
runs-on: ubuntu-22.04
if: failure() || cancelled()
needs:
- smoketests
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha || github.sha }}
context: E2E Tests/smoketests
description: E2E tests for mattermost server app
status: failure
update-success-final-status:
runs-on: ubuntu-22.04
if: success()
needs:
- smoketests
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha || github.sha }}
context: E2E Tests/smoketests
description: E2E tests for mattermost server app
status: success
e2e-smoketest:
uses: ./.github/workflows/e2e-tests-ci-template.yml
with:
ref: "${{ inputs.commit_sha || github.sha }}"
status_check_context: "E2E Tests/smoketests"

78
e2e-tests/.ci/.e2erc
View File

@ -1,37 +1,5 @@
# shellcheck disable=SC2148,SC2155
# Utility variables
# NB: these assume you `source` them from the directory this file is in
# Set up base docker compose file and export
export MME2E_DC_SERVER="docker compose -p mmserver -f ./server.yml"
export MME2E_DC_DASHBOARD="docker compose -p mmdashboard -f ./dashboard/docker/docker-compose.yml -f ./dashboard.override.yml"
export MME2E_UID=$(id -u)
export MME2E_OSTYPE=$(docker version -f '{{ .Client.Os }}')
export MME2E_ARCHTYPE=$(docker version -f '{{ .Client.Arch }}')
export NODE_VERSION_REQUIRED=$(cat ../../.nvmrc)
# Default values for optional variables
export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mattermost-enterprise-edition:$(git rev-parse --short=7 HEAD)"
export BROWSER_DEFAULT="chrome"
export SERVER_DEFAULT="self-hosted"
export TEST_DEFAULT="cypress"
export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
# OS specific defaults overrides
case $MME2E_OSTYPE in
darwin )
BROWSER_DEFAULT="electron" ;;
* )
esac
# Populate the optional variables that are used in the docker-compose file
export SERVER_IMAGE=${SERVER_IMAGE:-$SERVER_IMAGE_DEFAULT}
export BROWSER=${BROWSER:-$BROWSER_DEFAULT}
export SERVER=${SERVER:-$SERVER_DEFAULT}
export TEST=${TEST:-$TEST_DEFAULT}
export ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES:-$ENABLED_DOCKER_SERVICES_DEFAULT}
export TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT}
# Function definitions
mme2e_log () { echo "[$(date +%Y-%m-%dT%H:%M:%S%Z)]" "$@"; }
mme2e_get_current_shopt_arg () {
@ -98,10 +66,48 @@ mme2e_is_token_in_list() {
grep -qE "(^| )$TOKEN( |$)" <<<"$LIST"
}
# Utility alias, for interactive shell usage. Can be reversed with 'unalias docker-compose' in your shell
# NB: this won't work in the script
# Call prerequisite utility functions
mme2e_load_env_file
# Utility variables
# NB: these assume you `source` them from the directory this file is in
# Set up base docker compose file and export
export MME2E_DC_SERVER="docker compose -p mmserver -f ./server.yml"
export MME2E_DC_DASHBOARD="docker compose -p mmdashboard -f ./dashboard/docker/docker-compose.yml -f ./dashboard.override.yml"
export MME2E_UID=$(id -u)
export MME2E_OSTYPE=$(docker version -f '{{ .Client.Os }}')
export MME2E_ARCHTYPE=$(docker version -f '{{ .Client.Arch }}')
export NODE_VERSION_REQUIRED=$(cat ../../.nvmrc)
# Utility alias, for interactive shell usage. Can be reversed with 'unalias docker-compose-mmserver' in your shell
# NB: this only works in interactive shells
alias docker-compose-mmserver='${MME2E_DC_SERVER}'
alias docker-compose-mmdashboard='${MME2E_DC_DASHBOARD}'
# Call prerequisite utility functions
mme2e_load_env_file
# Default values for optional variables
export SERVER_IMAGE_DEFAULT="mattermostdevelopment/mattermost-enterprise-edition:$(git rev-parse --short=7 HEAD)"
export BROWSER_DEFAULT="chrome"
export SERVER_DEFAULT="self-hosted"
export TEST_DEFAULT="cypress"
export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
export BRANCH_DEFAULT=$(git branch --show-current || echo -n "unknown")
export BUILD_ID_DEFAULT=$(date +%s)
# OS specific defaults overrides
case $MME2E_OSTYPE in
darwin )
BROWSER_DEFAULT="electron" ;;
* )
esac
# Populate the optional variables that are used in the docker-compose file
export SERVER_IMAGE=${SERVER_IMAGE:-$SERVER_IMAGE_DEFAULT}
export BROWSER=${BROWSER:-$BROWSER_DEFAULT}
export SERVER=${SERVER:-$SERVER_DEFAULT}
export TEST=${TEST:-$TEST_DEFAULT}
export ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES:-$ENABLED_DOCKER_SERVICES_DEFAULT}
export TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT}
export BRANCH_UNSANITIZED=${BRANCH:-$BRANCH_DEFAULT}
export BRANCH=${BRANCH_UNSANITIZED::50} # The automation dashboard only accepts branch names up to 50 characters
export BUILD_ID_UNSANITIZED=${BUILD_ID:-$BUILD_ID_DEFAULT}
export BUILD_ID=${BUILD_ID_UNSANITIZED::64} # The automation dashboard only accepts build IDs up to 64 characters

6
e2e-tests/.ci/dashboard.generate_test_cycle.sh
View File

@ -11,7 +11,13 @@ fi
set -a
. .env.cypress
if [ -z "${AUTOMATION_DASHBOARD_URL:-}" ]; then
mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Skipping test cycle generation."
exit 0
fi
mme2e_log "Generating the test cycle on the Automation Dashboard"
cd ../cypress
npm i
# shellcheck disable=SC2086
exec node --trace-warnings generate_test_cycle.js $TEST_FILTER

65
e2e-tests/.ci/report.publish.sh Executable file
View File

@ -0,0 +1,65 @@
#!/bin/bash
# SC2034: <variable> appears unused.
# https://www.shellcheck.net/wiki/SC2034
# shellcheck disable=SC2034
set -e -u -o pipefail
cd "$(dirname "$0")"
. .e2erc
# Default required variables, assert that they are set, and document optional variables
: ${FULL_REPORT:=false} # Valid values: true, false
: ${TYPE:=NONE} # Valid values: PR, RELEASE, MASTER, MASTER_UNSTABLE, CLOUD, CLOUD_UNSTABLE, NONE (which is the same as omitting it)
: ${WEBHOOK_URL:-} # Optional. Mattermost webhook to post the report back to
: ${RELEASE_DATE:-} # Optional. If set, its value will be included in the report as the release date of the tested artifact
# Env vars used during the test. Their values will be included in the report
: ${BRANCH:?}
: ${BUILD_ID:?}
: ${MM_ENV:-}
# Populate intermediate variables
export BUILD_TAG="${SERVER_IMAGE##*/}"
export MM_DOCKER_IMAGE="${BUILD_TAG%%:*}" # NB: the 'mattermostdevelopment/' prefix is assumed
export MM_DOCKER_TAG="${BUILD_TAG##*:}"
export SERVER_TYPE="${SERVER}"
# NB: assume that BRANCH follows the convention 'server-pr-${PR_NUMBER}'. If multiple PRs match, the last one is used to generate the link
# Only needed if TYPE=PR
export PULL_REQUEST="https://github.com/mattermost/mattermost/pull/${BRANCH##*-}"
if [ -n "${TM4J_API_KEY:-}" ]; then
export TM4J_ENABLE=true
export JIRA_PROJECT_KEY=MM
export TM4J_ENVIRONMENT_NAME="${TEST}/${BROWSER}/${SERVER}"
case "${SERVER}" in
cloud)
export TM4J_FOLDER_ID="2014474" ;;
*)
export TM4J_FOLDER_ID="2014475" ;;
esac
: ${TEST_CYCLE_LINK_PREFIX:?}
: ${TM4J_CYCLE_KEY:-}
: ${TM4J_CYCLE_NAME:-}
mme2e_log "TMJ4 integration enabled."
fi
if [ -n "${DIAGNOSTIC_WEBHOOK_URL:-}" ]; then
: ${DIAGNOSTIC_USER_ID:?}
: ${DIAGNOSTIC_TEAM_ID:?}
mme2e_log "Diagnostic report upload enabled."
fi
if [ -n "${AWS_S3_BUCKET:-}" ]; then
: ${AWS_ACCESS_KEY_ID:?}
: ${AWS_SECRET_ACCESS_KEY:?}
mme2e_log "S3 report upload enabled."
fi
cd ../cypress/
if [ ! -d "results/" ]; then
mme2e_log "Error: 'results/' directory does not exist. Aborting report generation." >&2
exit 1
fi
npm i
node save_report.js

26
e2e-tests/.ci/server.generate.sh
View File

@ -61,7 +61,7 @@ services:
MM_TEAMSETTINGS_ENABLEOPENSERVER: "true"
MM_SQLSETTINGS_DATASOURCE: "postgres://mmuser:mostest@localhost:5432/mattermost_test?sslmode=disable&connect_timeout=10&binary_parameters=yes"
MM_SQLSETTINGS_DRIVERNAME: "postgres"
MM_EMAILSETTINGS_SMTPSERVER: "inbucket"
MM_EMAILSETTINGS_SMTPSERVER: "localhost"
MM_CLUSTERSETTINGS_READONLYCONFIG: "false"
MM_SERVICESETTINGS_ENABLEONBOARDINGFLOW: "false"
MM_FEATUREFLAGS_ONBOARDINGTOURTIPS: "false"
@ -259,9 +259,7 @@ EOL
generate_env_files() {
# Generate .env.server
mme2e_log "Generating .env.server"
mme2e_generate_envfile_from_var_names >.env.server <<-EOF
MM_LICENSE
EOF
truncate --size=0 .env.server
# Setting SERVER-specific variables
case "$SERVER" in
@ -282,16 +280,15 @@ generate_env_files() {
done
# Generating TEST-specific env files
BRANCH_DEFAULT=$(git branch --show-current)
BUILD_ID_DEFAULT=$(date +%s)
export BRANCH=${BRANCH:-$BRANCH_DEFAULT}
export BUILD_ID=${BUILD_ID:-$BUILD_ID_DEFAULT}
export CI_BASE_URL="${CI_BASE_URL:-localhost}"
# Some are defaulted in .e2erc due to being needed to other scripts as well
export CI_BASE_URL="${CI_BASE_URL:-http://localhost:8065}"
export REPO=mattermost # Static, but declared here for making generate_test_cycle.js easier to run
export HEADLESS=true # Static, but declared here for making generate_test_cycle.js easier to run
case "$TEST" in
cypress)
mme2e_log "Cypress: Generating .env.cypress"
truncate --size=0 .env.cypress
mme2e_generate_envfile_from_var_names >.env.cypress <<-EOF
BRANCH
BUILD_ID
@ -299,6 +296,7 @@ generate_env_files() {
BROWSER
HEADLESS
REPO
CYPRESS_pushNotificationServer
EOF
# Adding service-specific cypress variables
for SERVICE in $ENABLED_DOCKER_SERVICES; do
@ -327,8 +325,14 @@ generate_env_files() {
echo "CYPRESS_serverEdition=E20" >>.env.cypress
;;
esac
# If the dashboard is running, load .env.dashboard into .env.cypress
if DC_COMMAND="$MME2E_DC_DASHBOARD" mme2e_wait_service_healthy dashboard 1; then
# Add Automation Dashboard related variables to cypress container
if [ -n "${AUTOMATION_DASHBOARD_URL:-}" ]; then
mme2e_log "Automation dashboard URL is set: loading related variables into the Cypress container"
mme2e_generate_envfile_from_var_names >>.env.cypress <<-EOF
AUTOMATION_DASHBOARD_URL
AUTOMATION_DASHBOARD_TOKEN
EOF
elif DC_COMMAND="$MME2E_DC_DASHBOARD" mme2e_wait_service_healthy dashboard 1; then
mme2e_log "Detected a running automation dashboard: loading its access variables into the Cypress container"
cat >>.env.cypress <.env.dashboard
fi

7
e2e-tests/.ci/server.prepare.sh
View File

@ -3,6 +3,13 @@ set -e -u -o pipefail
cd "$(dirname "$0")"
. .e2erc
if [ -n "${MM_LICENSE:-}" ]; then
# We prefer uploading the license here, instead of setting the env var for the server
# This is to retain the flexibility of being able to remove it programmatically, if the tests require it
mme2e_log "Uploading license to server"
${MME2E_DC_SERVER} exec -T -- server mmctl --local license upload-string "$MM_LICENSE"
fi
if [ "$TEST" = "cypress" ]; then
mme2e_log "Prepare Cypress: install dependencies"
${MME2E_DC_SERVER} exec -T -u 0 -- cypress bash -c "id $MME2E_UID || useradd -u $MME2E_UID -m nodeci" # Works around the node image's assumption that the app files are owned by user 1000

4
e2e-tests/Makefile
View File

@ -39,6 +39,10 @@ cloud-init: requirecmd-jq requirecmd-curl
cloud-teardown:
bash ./.ci/server.cloud_teardown.sh
.PHONY: publish-report
publish-report: requirecmd-node
bash ./.ci/report.publish.sh
.PHONY: fmt-node fmt-shell fmt
requirecmd-%:
@which "$(*)" >/dev/null || { echo "Error, missing required CLI tool: $(*). Aborting." >&2; exit 1; }

2
e2e-tests/README.md
View File

@ -38,6 +38,7 @@ Instructions, detailed:
* When running with `SERVER=cloud`, this will automatically create a cloud customer against the specified `CWS_URL` service, and delete that user after the run is complete.
* If you want to run the Playwright tests instead of the Cypress ones, you can run `TEST=playwright make`
* If you just want to run a local server instance, without any further testing, you can run `TEST=none make`
* If you're using the automation dashboard, you have the option of sharding the E2E test run: you can launch the `make` command in parallel on different machiness (NB: you must use the same `BUILD_ID` and `BRANCH` values that you used for `make generate-test-cycle`) to distribute running the test cases across them. When doing this, you should also set on each machine the `CI_BASE_URL` variable to a value that uniquely identifies the instance where `make` is running.
4. `make stop`: tears down the server (and the dashboard, if running)
* `make clean` will also remove any generated environment or docker-compose files, in addition to stopping the containers.
@ -50,6 +51,7 @@ Notes:
* If their value is fixed (e.g. a static server configuration), these may be simply added to the `docker_compose_generator.sh` file, to the appropriate container.
* If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir, and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability, and simplifies the question of what container has access to which variable considerably.
* Exceptions are of course accepted wherever it makes sense (e.g. if you need to group variables based on some common functionality)
- The `publish-report` Make target is meant for internal usage. Usage and variables are documented in the respective scripts.
##### For code changes:
* `make fmt-ci` to format and check yaml files and shell scripts.

4
e2e-tests/cypress/tests/integration/channels/plugins/plugin_startup_fail_spec.js
View File

@ -33,8 +33,8 @@ describe('If plugins fail to start, they can be disabled', () => {
it('MM-T2391 If plugins fail to start, they can be disabled', () => {
const mimeType = 'application/gzip';
cy.fixture(gitlabPlugin.filename, 'binary').
then(Cypress.Blob.binaryStringToBlob).
cy.fixture(gitlabPlugin.filename, null).
then(Cypress.Blob.arrayBufferToBlob).
then((fileContent) => {
cy.get('input[type=file]').attachFile({fileContent, fileName: gitlabPlugin.filename, mimeType});
});

2
e2e-tests/cypress/utils/report.js
View File

@ -246,7 +246,7 @@ function generateTitle() {
let dockerImageLink = '';
if (MM_DOCKER_IMAGE && MM_DOCKER_TAG) {
dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermost/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`;
dockerImageLink = ` with [${MM_DOCKER_IMAGE}:${MM_DOCKER_TAG}](https://hub.docker.com/r/mattermostdevelopment/${MM_DOCKER_IMAGE}/tags?name=${MM_DOCKER_TAG})`;
}
let releaseDate = '';