IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

Support release testing (#27587)

Browse Source 
* Support release testing
* Merge resolve-ref and generate-test-variables jobs
This commit is contained in:
Mario Vitale 2024-07-10 16:47:26 +02:00 committed by GitHub
parent 5d7027a172
commit 731f056f54
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 93 additions and 72 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

147
.github/workflows/e2e-fulltests-ci.yml vendored
View File

@ -7,7 +7,7 @@ on:
inputs:
ref:
type: string
description: Branch or SHA of the commit to test. Ignored if PR_NUMBER is specified
description: Branch, tag or SHA of the commit to test. Ignored if PR_NUMBER is specified. Set to a tag when release testing.
required: false
PR_NUMBER:
type: string
@ -22,6 +22,7 @@ on:
options:
- PR
- RELEASE
- RELEASE_CLOUD
- MASTER
- MASTER_UNSTABLE
- CLOUD
@ -34,41 +35,8 @@ concurrency:
cancel-in-progress: true
jobs:
resolve-ref:
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
outputs:
commit_sha: "${{ steps.resolve-ref.outputs.commit_sha }}"
BRANCH: "${{ steps.resolve-ref.outputs.BRANCH }}"
env:
PR_NUMBER: "${{ inputs.PR_NUMBER }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: "${{ inputs.ref || github.sha }}"
fetch-depth: 0
- name: ci/resolve-ref
id: resolve-ref
run: |
set -e
if [ -n "$PR_NUMBER" ]; then
curl -fsSL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer ${{ github.token }}" "${{ github.api_url }}/repos/${{ github.repository }}/pulls/${PR_NUMBER}" -o pr.json
echo "commit_sha=$(jq -r .head.sha <pr.json)" >> $GITHUB_OUTPUT
echo "BRANCH=server-pr-${PR_NUMBER}" >> $GITHUB_OUTPUT
# For reference, the real branch name may be retrievable as follows:
#echo "BRANCH=$(jq -r .head.ref <pr.json)" >> $GITHUB_OUTPUT
else
echo "commit_sha=$(git rev-parse --verify HEAD)" >> $GITHUB_OUTPUT
echo "BRANCH=${{ inputs.ref || github.sha }}" >> $GITHUB_OUTPUT
fi
generate-test-variables:
runs-on: ubuntu-22.04
needs:
- resolve-ref
permissions:
issues: write
pull-requests: write
@ -76,6 +44,9 @@ jobs:
run:
shell: bash
outputs:
commit_sha: "${{ steps.generate.outputs.commit_sha }}"
BRANCH: "${{ steps.generate.outputs.BRANCH }}"
SERVER_IMAGE: "${{ steps.generate.outputs.SERVER_IMAGE }}"
status_check_context: "${{ steps.generate.outputs.status_check_context }}"
workers_number: "${{ steps.generate.outputs.workers_number }}"
server_uppercase: "${{ steps.generate.outputs.server_uppercase }}" # Required for license selection
@ -84,8 +55,10 @@ jobs:
TEST_FILTER: "${{ steps.generate.outputs.TEST_FILTER }}"
BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
TM4J_ENABLE: "${{ steps.generate.outputs.TM4J_ENABLE }}"
REPORT_TYPE: "${{ steps.generate.outputs.REPORT_TYPE }}"
env:
GH_TOKEN: "${{ github.token }}"
REF: "${{ inputs.ref || github.sha }}"
PR_NUMBER: "${{ inputs.PR_NUMBER || '' }}"
REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
# We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
@ -94,62 +67,112 @@ jobs:
--excludeGroup="@te_only,@cloud_only,@high_availability"
--sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap"
--sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
TEST_FILTER_DAILY_ONPREM: >-
TEST_FILTER_PROD_ONPREM: >-
--stage="@prod"
--excludeGroup="@te_only,@cloud_only,@high_availability"
--sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
--sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
TEST_FILTER_DAILY_CLOUD: >-
TEST_FILTER_PROD_CLOUD: >-
--stage="@prod"
--excludeGroup="@not_cloud,@cloud_trial,@e20_only,@te_only,@high_availability,@license_removal"
--sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap,@playbooks"
--sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa"
MM_ENV: "${{ inputs.MM_ENV || '' }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: "${{ inputs.ref || github.sha }}"
fetch-depth: 0
- name: ci/generate-test-variables
id: generate
run: |
COMMIT_SHA="${{needs.resolve-ref.outputs.commit_sha}}"
# BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
# Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
BUILD_ID_PREFIX="${{ github.run_id }}_${{ github.run_attempt }}-${COMMIT_SHA::7}"
MM_ENV_HASH=$(md5sum -z <<<"$MM_ENV" | cut -c-8)
SERVER=onprem
if grep -q CLOUD <<<"$REPORT_TYPE"; then
SERVER=cloud
else
SERVER=onprem
fi
case "$REPORT_TYPE" in
NONE | PR)
echo "status_check_context=E2E Tests/test${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
echo "workers_number=20" >> $GITHUB_OUTPUT
echo "TEST_FILTER=$TEST_FILTER_PR" >> $GITHUB_OUTPUT
echo "BUILD_ID=${BUILD_ID_PREFIX}-${REPORT_TYPE@L}-${SERVER}-ent" >> $GITHUB_OUTPUT
### For ref generation: either use 'inputs.PR_NUMBER' or 'inputs.ref'
if [ -n "$PR_NUMBER" ]; then
curl -fsSL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer ${{ github.token }}" "${{ github.api_url }}/repos/${{ github.repository }}/pulls/${PR_NUMBER}" -o pr.json
COMMIT_SHA="$(jq -r .head.sha <pr.json)"
BRANCH="server-pr-${PR_NUMBER}" # For reference, the real branch name may be retrievable with command: 'jq -r .head.ref <pr.json'
else
COMMIT_SHA="$(git rev-parse --verify HEAD)"
BRANCH="${REF}" # Note that REF is not necessarily a branch name, but this is ok since it's only used as an identifier against the Automation Dashboard
fi
SERVER_IMAGE_TAG="${COMMIT_SHA::7}"
SERVER_IMAGE_ORG=mattermostdevelopment
BUILD_ID_SUFFIX="${REPORT_TYPE@L}-${SERVER}-ent"
STATUS_CHECK_CONTEXT="E2E Tests/test${MM_ENV:+/$MM_ENV_HASH}"
WORKERS_NUMBER=20
TEST_FILTER="$TEST_FILTER_PR"
COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
### Run sanity assertions after variable generations
[ "$REPORT_TYPE" != "PR" -o -n "$PR_NUMBER" ] # If using 'REPORT_TYPE=PR', 'PR_NUMBER' must be set
;;
MASTER | MASTER_UNSTABLE | CLOUD | CLOUD_UNSTABLE)
IS_TEST_UNSTABLE=$(sed -n -E 's/^.*(UNSTABLE).*$/\1/p' <<< "$REPORT_TYPE") # Variable is UNSTABLE if report type is for unstable tests
if grep -q CLOUD <<<$REPORT_TYPE; then
SERVER=cloud
fi
TEST_FILTER_VARIABLE="TEST_FILTER_DAILY_${SERVER@U}"
BUILD_ID_SUFFIX="${IS_TEST_UNSTABLE:+unstable-}daily-${SERVER}-ent"
echo "status_check_context=E2E Tests/test-${BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}" >> $GITHUB_OUTPUT
echo "workers_number=10" >> $GITHUB_OUTPUT # Daily tests are not time critical, and it's more efficient to run on fewer workers
echo "TEST_FILTER=${!TEST_FILTER_VARIABLE} ${IS_TEST_UNSTABLE:+--invert}" >> $GITHUB_OUTPUT
echo "BUILD_ID=${BUILD_ID_PREFIX}-${BUILD_ID_SUFFIX}" >> $GITHUB_OUTPUT
echo "TM4J_ENABLE=true" >> $GITHUB_OUTPUT
### Populate support variables
_IS_TEST_UNSTABLE=$(sed -n -E 's/^.*(UNSTABLE).*$/\1/p' <<< "$REPORT_TYPE") # The variable's value is 'UNSTABLE' if report type is for unstable tests, otherwise it's empty
_TEST_FILTER_VARIABLE="TEST_FILTER_PROD_${SERVER@U}"
### For ref generation: ignore both 'inputs.PR_NUMBER' and 'inputs.ref', and use master branch directly. Note that the commit_sha is still used for reporting the test result, and for the testing scripts and test cases
COMMIT_SHA="$(git rev-parse --verify origin/master)"
BRANCH=master
SERVER_IMAGE_TAG=master
SERVER_IMAGE_ORG=mattermostdevelopment
BUILD_ID_SUFFIX="${_IS_TEST_UNSTABLE:+unstable-}daily-${SERVER}-ent"
STATUS_CHECK_CONTEXT="E2E Tests/test-${BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}"
WORKERS_NUMBER=10 # Daily tests are not time critical, and it's more efficient to run on fewer workers
TEST_FILTER="${!_TEST_FILTER_VARIABLE} ${_IS_TEST_UNSTABLE:+--invert}"
TM4J_ENABLE=true
COMPUTED_REPORT_TYPE="${REPORT_TYPE}"
;;
RELEASE | RELEASE_CLOUD)
### Populate support variables
_TEST_FILTER_VARIABLE="TEST_FILTER_PROD_${SERVER@U}"
### For ref generation: assume the 'inputs.ref' is a tag, and use the first two digits to construct the branch name
COMMIT_SHA="$(git rev-parse --verify HEAD)"
BRANCH=$(sed -E "s/v([0-9]+)\.([0-9]+)\..+$/release-\1.\2/g" <<<$REF)
SERVER_IMAGE_TAG="$(cut -c2- <<<$REF)" # Remove the leading 'v' from the given tag name, to generate the docker image tag
SERVER_IMAGE_ORG=mattermost
BUILD_ID_SUFFIX="release-${SERVER}-ent"
STATUS_CHECK_CONTEXT="E2E Tests/test-${BUILD_ID_SUFFIX}${MM_ENV:+/$MM_ENV_HASH}"
WORKERS_NUMBER=20
TEST_FILTER="${!_TEST_FILTER_VARIABLE}"
TM4J_ENABLE=true
COMPUTED_REPORT_TYPE=RELEASE
### Run sanity assertions after variable generations
git show-ref --verify "refs/tags/${REF}" # 'inputs.ref' must be a tag, for release report types
git show-ref --verify "refs/remotes/origin/${BRANCH}" # The release branch computed from the given tag must exist
;;
*)
# TODO implement release testing, in the future
echo "Fatal: unimplemented test type. Aborting."
exit 1
esac
echo "commit_sha=${COMMIT_SHA}" >> $GITHUB_OUTPUT
echo "BRANCH=${BRANCH}" >> $GITHUB_OUTPUT
echo "SERVER_IMAGE=${SERVER_IMAGE_ORG}/mattermost-enterprise-edition:${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
echo "SERVER=${SERVER}" >> $GITHUB_OUTPUT
echo "server_uppercase=${SERVER@U}" >> $GITHUB_OUTPUT
echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT
echo "status_check_context=${STATUS_CHECK_CONTEXT}" >> $GITHUB_OUTPUT
echo "workers_number=${WORKERS_NUMBER}" >> $GITHUB_OUTPUT
echo "TEST_FILTER=${TEST_FILTER}" >> $GITHUB_OUTPUT
echo "TM4J_ENABLE=${TM4J_ENABLE:-}" >> $GITHUB_OUTPUT
echo "REPORT_TYPE=${COMPUTED_REPORT_TYPE}" >> $GITHUB_OUTPUT
# BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
# Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
echo "BUILD_ID=${{ github.run_id }}_${{ github.run_attempt }}-${SERVER_IMAGE_TAG}-${BUILD_ID_SUFFIX}" >> $GITHUB_OUTPUT
- name: ci/notify-user
run: |
if [ -n "$PR_NUMBER" ]; then
WORKFLOW_RUN_URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
CYCLE_URL="https://automation-dashboard.vercel.app/cycle/${{ github.run_id }}"
gh issue -R "${{ github.repository }}" comment "$PR_NUMBER" --body-file - <<EOF
E2E test run is starting for commit \`${{ needs.resolve-ref.outputs.commit_sha }}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
E2E test run is starting for commit \`${{ steps.generate.outputs.commit_sha }}\`${MM_ENV:+, with \`MM_ENV=$MM_ENV\`}.
You can check its progress by either:
- Looking at the corresponding commit status, which will be available in a few moments: \`${{ steps.generate.outputs.status_check_context }}\`.
- Looking at the [E2E test's Workflow Run]($WORKFLOW_RUN_URL), with Run ID \`${{ github.run_id }}\`.
@ -159,7 +182,6 @@ jobs:
e2e-fulltest:
needs:
- resolve-ref
- generate-test-variables
uses: ./.github/workflows/e2e-tests-ci-template.yml
strategy:
@ -167,19 +189,20 @@ jobs:
type:
- name: PR
with:
commit_sha: "${{ needs.resolve-ref.outputs.commit_sha }}"
commit_sha: "${{ needs.generate-test-variables.outputs.commit_sha }}"
status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}"
workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}"
testcase_failure_fatal: false
run_preflight_checks: false
enable_reporting: true
SERVER: "${{ needs.generate-test-variables.outputs.SERVER }}"
SERVER_IMAGE: "${{ needs.generate-test-variables.outputs.SERVER_IMAGE }}"
ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER }}"
MM_ENV: "${{ inputs.MM_ENV || '' }}"
BRANCH: "${{ needs.resolve-ref.outputs.BRANCH }}"
BRANCH: "${{ needs.generate-test-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
REPORT_TYPE: "${{ needs.generate-test-variables.outputs.REPORT_TYPE }}"
secrets:
MM_LICENSE: "${{ secrets[format('MM_E2E_TEST_LICENSE_{0}_ENT', needs.generate-test-variables.outputs.server_uppercase)] }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"

14
.github/workflows/e2e-tests-ci-template.yml vendored
View File

@ -34,6 +34,9 @@ on:
type: string # Valid values are: onprem, cloud
required: false
default: onprem
SERVER_IMAGE:
type: string
required: false
ENABLED_DOCKER_SERVICES:
type: string
required: false
@ -163,7 +166,6 @@ jobs:
shell: bash
outputs:
workers: "${{ steps.generate.outputs.workers }}"
SERVER_IMAGE: "${{ steps.generate.outputs.SERVER_IMAGE }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
@ -174,15 +176,9 @@ jobs:
id: generate
env:
WORKERS: ${{ inputs.workers_number }}
REPORT_TYPE: ${{ inputs.REPORT_TYPE }}
run: |
[ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0
echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT
if grep -qE '^((MASTER|CLOUD)(_UNSTABLE)?)$' <<<"$REPORT_TYPE"; then
# For MASTER, MASTER_UNSTABLE, CLOUD and CLOUD_UNSTABLE runs, always use the latest master image
# The commit_sha is still used for reporting the test result, and for the testing scripts and test cases
echo "SERVER_IMAGE=mattermostdevelopment/mattermost-enterprise-edition:master" >> $GITHUB_OUTPUT
fi
generate-test-cycle:
runs-on: ubuntu-22.04
@ -256,7 +252,7 @@ jobs:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
SERVER: "${{ inputs.SERVER }}"
SERVER_IMAGE: "${{ needs.generate-build-variables.outputs.SERVER_IMAGE }}"
SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
MM_LICENSE: "${{ secrets.MM_LICENSE }}"
ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER: "${{ inputs.TEST_FILTER }}"
@ -346,7 +342,7 @@ jobs:
env:
TYPE: "${{ inputs.REPORT_TYPE }}"
SERVER: "${{ inputs.SERVER }}"
SERVER_IMAGE: "${{ needs.generate-build-variables.outputs.SERVER_IMAGE }}"
SERVER_IMAGE: "${{ inputs.SERVER_IMAGE }}"
WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}"
BRANCH: "${{ inputs.BRANCH }}"
BUILD_ID: "${{ inputs.BUILD_ID }}"

2
e2e-tests/.ci/server.generate.sh
View File

@ -315,6 +315,8 @@ generate_env_files() {
case "$SERVER" in
cloud)
echo "CYPRESS_serverEdition=Cloud" >>.env.cypress
echo "CYPRESS_cwsURL=${CWS_URL}" >> .env.cypress
echo "CYPRESS_cwsAPIURL=${CWS_URL}" >> .env.cypress
;;
*)
echo "CYPRESS_serverEdition=E20" >>.env.cypress

2
e2e-tests/.ci/server.start.sh
View File

@ -9,7 +9,7 @@ ${MME2E_DC_SERVER} down -v --remove-orphans
# Wait for the required server image
mme2e_log "Waiting for server image to be available"
mme2e_wait_image "$SERVER_IMAGE" 30 60
mme2e_wait_image "$SERVER_IMAGE" 4 30
# Launch mattermost-server, and wait for it to be healthy
mme2e_log "Starting E2E containers"