IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

CLD-798 Add support for MM_ENV, various fixes (#26616)

Browse Source 
* Add support for MM_ENV, various fixes
* Fix workflow template
* Fix reporting steps ordering
* Fix typo
* Improve state check message description
* Fix workflow syntax
* Fix fulltest variable generation
* Fix PR_NUMBER variable usage
* Fix BUILD_ID, add dashboard URL to status check
* Fix automation dashboard URL

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
This commit is contained in:
Mario Vitale 2024-04-03 12:14:19 +02:00 committed by GitHub
parent 1f3bc992ea
commit dc36e42448
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 124 additions and 94 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
.github/workflows/e2e-fulltests-ci.yml vendored
View File

@ -6,6 +6,14 @@ on:
workflow_dispatch:
inputs:
ref:
type: string
description: Branch or SHA of the commit to test. Ignored if PR_NUMBER is specified
required: false
PR_NUMBER:
type: string
description: If testing a PR, specify this instead of ref
required: false
MM_ENV:
type: string
required: false
REPORT_TYPE:
@ -22,13 +30,48 @@ on:
default: NONE
jobs:
resolve-ref:
runs-on: ubuntu-22.04
defaults:
run:
shell: bash
outputs:
commit_sha: "${{ steps.resolve-ref.outputs.commit_sha }}"
BRANCH: "${{ steps.resolve-ref.outputs.BRANCH }}"
env:
PR_NUMBER: "${{ inputs.PR_NUMBER }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: "${{ inputs.ref || github.sha }}"
fetch-depth: 0
- name: ci/resolve-ref
id: resolve-ref
run: |
set -e
if [ -n "$PR_NUMBER" ]; then
curl -fsSL -H "Accept: application/vnd.github+json" -H "X-GitHub-Api-Version: 2022-11-28" -H "Authorization: Bearer ${{ github.token }}" "${{ github.api_url }}/repos/${{ github.repository }}/pulls/${PR_NUMBER}" -o pr.json
echo "commit_sha=$(jq -r .head.sha <pr.json)" >> $GITHUB_OUTPUT
echo "BRANCH=server-pr-${PR_NUMBER}" >> $GITHUB_OUTPUT
#echo "BRANCH=$(jq -r .head.ref <pr.json)" >> $GITHUB_OUTPUT
else
echo "commit_sha=$(git rev-parse --verify HEAD)" >> $GITHUB_OUTPUT
echo "BRANCH=${{ inputs.ref || github.sha }}" >> $GITHUB_OUTPUT
fi
generate-test-variables:
runs-on: ubuntu-22.04
needs:
- resolve-ref
defaults:
run:
shell: bash
outputs:
status_check_context: "${{ steps.generate.outputs.status_check_context }}"
workers_number: "${{ steps.generate.outputs.workers_number }}"
ENABLED_DOCKER_SERVICES: "${{ steps.generate.outputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER: "${{ steps.generate.outputs.TEST_FILTER }}"
BUILD_ID: "${{ steps.generate.outputs.BUILD_ID }}"
env:
# We could exclude the @smoke group for PRs, but then we wouldn't have it in the report
TEST_FILTER_PR: >-
@ -39,14 +82,18 @@ jobs:
steps:
- name: ci/generate-test-variables
id: generate
shell: bash
run: |
COMMIT_SHA="${{needs.resolve-ref.outputs.commit_sha}}"
# BUILD_ID format: $pipelineID-$imageTag-$testType-$serverType-$serverEdition
# Reference on BUILD_ID parsing: https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23
BUILD_ID_PREFIX="${{ github.run_id }}_${{ github.run_attempt }}-${COMMIT_SHA::7}"
case "${{ inputs.REPORT_TYPE }}" in
NONE | PR)
echo "status_check_context=E2E Tests/test" >> $GITHUB_OUTPUT
echo "workers_number=20" >> $GITHUB_OUTPUT
echo "ENABLED_DOCKER_SERVICES=postgres inbucket minio openldap elasticsearch keycloak" >> $GITHUB_OUTPUT
echo "TEST_FILTER=$TEST_FILTER_PR" >> $GITHUB_OUTPUT
echo "BUILD_ID=${BUILD_ID_PREFIX}-pr-onprem-ent" >> $GITHUB_OUTPUT
;;
*)
# TODO implement other test types, in the future
@ -56,6 +103,7 @@ jobs:
e2e-fulltest:
needs:
- resolve-ref
- generate-test-variables
uses: ./.github/workflows/e2e-tests-ci-template.yml
strategy:
@ -63,7 +111,7 @@ jobs:
type:
- name: PR
with:
ref: "${{ inputs.ref || github.sha }}"
commit_sha: "${{ needs.resolve-ref.outputs.commit_sha }}"
status_check_context: "${{ needs.generate-test-variables.outputs.status_check_context }}"
workers_number: "${{ needs.generate-test-variables.outputs.workers_number }}"
testcase_failure_fatal: false
@ -71,6 +119,9 @@ jobs:
enable_reporting: true
ENABLED_DOCKER_SERVICES: "${{ needs.generate-test-variables.outputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER: "${{ needs.generate-test-variables.outputs.TEST_FILTER }}"
MM_ENV: "${{ inputs.MM_ENV || '' }}"
BRANCH: "${{ needs.resolve-ref.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-test-variables.outputs.BUILD_ID }}"
REPORT_TYPE: "${{ inputs.REPORT_TYPE }}"
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"

150
.github/workflows/e2e-tests-ci-template.yml vendored
View File

@ -6,7 +6,7 @@ on:
# NB: this does not support using branch names that belong to forks.
# In those cases, you should specify directly the commit SHA that you want to test, or
# some wrapper workflow that does it for you (e.g. the slash command for initiating a PR test)
ref:
commit_sha:
type: string
required: true
status_check_context:
@ -39,6 +39,12 @@ on:
MM_ENV:
type: string
required: false
BRANCH:
type: string
required: false
BUILD_ID:
type: string
required: false
REPORT_TYPE:
type: string
required: false
@ -59,32 +65,15 @@ on:
required: false
jobs:
resolve-ref:
runs-on: ubuntu-22.04
outputs:
commit_sha: "${{ steps.resolve-ref.outputs.commit_sha }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
fetch-depth: 0
- name: ci/resolve-ref
id: resolve-ref
run: |
echo "commit_sha=$(git rev-parse --verify HEAD)" >> $GITHUB_OUTPUT
update-initial-status:
runs-on: ubuntu-22.04
needs:
- resolve-ref
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: E2E tests for mattermost server app
status: pending
@ -101,7 +90,7 @@ jobs:
if: "${{ inputs.run_preflight_checks }}"
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-node
if: "${{ inputs.run_preflight_checks }}"
@ -132,7 +121,7 @@ jobs:
if: "${{ inputs.run_preflight_checks }}"
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-node
if: "${{ inputs.run_preflight_checks }}"
@ -160,17 +149,14 @@ jobs:
generate-build-variables:
runs-on: ubuntu-22.04
needs:
- resolve-ref
- update-initial-status
outputs:
workers: "${{ steps.workers.outputs.workers }}"
BRANCH: "${{ steps.branch.outputs.BRANCH }}"
BUILD_ID: "${{ steps.build-id.outputs.BUILD_ID }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/generate-workers
id: workers
@ -179,44 +165,22 @@ jobs:
run: |
[ "$WORKERS" -gt "0" ] # Assert that the workers number is an integer greater than 0
echo "workers="$(jq --slurp --compact-output '[range('"$WORKERS"')] | map(tostring)' /dev/null) >> $GITHUB_OUTPUT
- name: ci/generate-branch
id: branch
run: |
PR_LIST=$(curl -L -H "Accept: application/vnd.github+json" \
-H "Authorization: Bearer ${{ github.token }}" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"${{ github.api_url }}/repos/${{ github.repository }}/commits/${{ needs.resolve-ref.outputs.commit_sha }}/pulls" 2>/dev/null || echo -n "")
PR_NAMES_LIST=$(echo -n "$PR_LIST" | jq -r 'map(.number) | join("-")')
if [ -n "$PR_NAMES_LIST" ]; then
# This commit belongs to one or more PRs.
# Prefer displaying the PR names in the Automation Dashboard, instead of the branch name
BRANCH="server-pr-${PR_NAMES_LIST}"
else
# This commit does not belong to a PR. Use the given input ref as-is
BRANCH="${{ inputs.ref }}"
fi
echo "Generated branch identifier for E2E run: ${BRANCH}"
echo "BRANCH=${BRANCH}" >> "$GITHUB_OUTPUT"
- name: ci/calculate-build-id
id: build-id
run: |
BUILD_ID="$(git rev-parse --short=7 HEAD)-${{ github.run_id }}-${{ github.run_attempt }}-ent"
echo "Generated BUILD_ID for E2E run: ${BUILD_ID}"
echo "BUILD_ID=${BUILD_ID}" >> "$GITHUB_OUTPUT"
generate-test-cycle:
runs-on: ubuntu-22.04
needs:
- generate-build-variables
- update-initial-status
defaults:
run:
shell: bash
working-directory: e2e-tests
outputs:
status_check_url: "${{ steps.e2e-test-gencycle.outputs.status_check_url }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-node
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@ -226,14 +190,23 @@ jobs:
cache: npm
cache-dependency-path: "e2e-tests/cypress/package-lock.json"
- name: ci/e2e-test-gencycle
id: e2e-test-gencycle
env:
AUTOMATION_DASHBOARD_URL: "${{ secrets.AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.AUTOMATION_DASHBOARD_TOKEN }}"
BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}"
BRANCH: "${{ inputs.BRANCH }}"
BUILD_ID: "${{ inputs.BUILD_ID }}"
TEST_FILTER: "${{ inputs.TEST_FILTER }}"
run: |
make generate-test-cycle
set -e -o pipefail
make generate-test-cycle | tee generate-test-cycle.out
# Extract cycle's dashboard URL, if present
TEST_CYCLE_ID=$(sed -nE "s/^.*id: '([^']+)'.*$/\1/p" <generate-test-cycle.out)
if [ -n "$TEST_CYCLE_ID" ]; then
echo "status_check_url=https://automation-dashboard.vercel.app/cycles/${TEST_CYCLE_ID}" >> $GITHUB_OUTPUT
else
echo "status_check_url=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_OUTPUT
fi
test:
continue-on-error: true # Individual runner failures shouldn't prevent the completion of an E2E run
@ -268,15 +241,15 @@ jobs:
ENABLED_DOCKER_SERVICES: "${{ inputs.ENABLED_DOCKER_SERVICES }}"
TEST_FILTER: "${{ inputs.TEST_FILTER }}"
MM_ENV: "${{ inputs.MM_ENV }}"
BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}"
BRANCH: "${{ inputs.BRANCH }}"
BUILD_ID: "${{ inputs.BUILD_ID }}"
CI_BASE_URL: "http://localhost:8065/?worker_index=${{ matrix.worker_index }}"
CYPRESS_pushNotificationServer: "${{ secrets.PUSH_NOTIFICATION_SERVER }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-macos-docker
if: runner.os == 'macos'
@ -304,30 +277,39 @@ jobs:
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
- name: ci/e2e-test-assert-results
if: ${{ inputs.testcase_failure_fatal }}
run: |
# Assert that the run contained 0 failures
CYPRESS_FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
echo "Cypress run completed with $CYPRESS_FAILURES failures"
[ "$CYPRESS_FAILURES" = "0" ]
publish-report:
report:
runs-on: ubuntu-22.04
needs:
- generate-build-variables
- test
defaults:
run:
shell: bash
working-directory: e2e-tests
outputs:
failures: "${{ steps.calculate-failures.outputs.failures }}"
steps:
- name: ci/checkout-repo
if: "${{ inputs.enable_reporting }}"
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
with:
ref: ${{ inputs.ref }}
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/download-artifacts
uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/report-calculate-failures
id: calculate-failures
run: |
FAILURES=$(find cypress/results -name '*.json' | xargs -l jq -r '.stats.failures' | jq -s add)
echo "failures=${FAILURES}" >> $GITHUB_OUTPUT
echo "Cypress run completed with $FAILURES failures"
- name: ci/e2e-test-assert-results
if: "${{ inputs.testcase_failure_fatal }}"
run: |
# Assert that the run contained 0 failures
[ "${{ steps.calculate-failures.outputs.failures }}" = "0" ]
- name: ci/setup-node
if: "${{ inputs.enable_reporting }}"
uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
@ -336,19 +318,13 @@ jobs:
node-version-file: ".nvmrc"
cache: npm
cache-dependency-path: "e2e-tests/cypress/package-lock.json"
- name: ci/download-artifacts
if: "${{ inputs.enable_reporting }}"
uses: actions/download-artifact@eaceaf801fd36c7dee90939fad912460b18a1ffe # v4.1.2
with:
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/publish-report
if: "${{ inputs.enable_reporting }}"
env:
TYPE: "${{ inputs.REPORT_TYPE }}"
WEBHOOK_URL: "${{ secrets.REPORT_WEBHOOK_URL }}"
BRANCH: "${{ needs.generate-build-variables.outputs.BRANCH }}"
BUILD_ID: "${{ needs.generate-build-variables.outputs.BUILD_ID }}"
BRANCH: "${{ inputs.BRANCH }}"
BUILD_ID: "${{ inputs.BUILD_ID }}"
MM_ENV: "${{ inputs.MM_ENV }}"
TM4J_API_KEY: "${{ secrets.REPORT_TM4J_API_KEY }}"
TEST_CYCLE_LINK_PREFIX: "${{ secrets.REPORT_TM4J_TEST_CYCLE_LINK_PREFIX }}"
@ -359,32 +335,38 @@ jobs:
runs-on: ubuntu-22.04
if: failure() || cancelled()
needs:
- resolve-ref
- publish-report
- generate-test-cycle
- test
- report
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: E2E tests for mattermost server app
description: |
Completed with ${{ needs.report.outputs.failures }} failures
status: failure
target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}"
update-success-final-status:
runs-on: ubuntu-22.04
if: success()
needs:
- resolve-ref
- publish-report
- generate-test-cycle
- test
- report
steps:
- uses: mattermost/actions/delivery/update-commit-status@main
env:
GITHUB_TOKEN: ${{ github.token }}
with:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ needs.resolve-ref.outputs.commit_sha }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.status_check_context }}
description: E2E tests for mattermost server app
description: |
Completed with ${{ needs.report.outputs.failures }} failures
status: success
target_url: "${{ needs.generate-test-cycle.outputs.status_check_url }}"

4
.github/workflows/e2e-tests-ci.yml vendored
View File

@ -13,5 +13,5 @@ jobs:
e2e-smoketest:
uses: ./.github/workflows/e2e-tests-ci-template.yml
with:
ref: "${{ inputs.commit_sha || github.sha }}"
status_check_context: "E2E Tests/smoketests"
commit_sha: "${{ inputs.commit_sha || github.sha }}"
status_check_context: "E2E Tests/smoketests"

4
e2e-tests/Makefile
View File

@ -1,7 +1,3 @@
# Load environment variables from .env file
-include .ci/.env
export
SHELL := /bin/bash
.PHONY: all run stop clean

5
e2e-tests/README.md
View File

@ -17,7 +17,7 @@ Instructions, tl;dr: create a local branch with your E2E test changes, then open
Instructions, detailed:
1. (optional, undefined variables are set to sane defaults) Create the `.ci/env` file, and populate it with the variables you need out of the following list:
* `SERVER`: either `self-hosted` (default) or `cloud`.
* `CWS_URL` (mandatory when `SERVER=cloud`, only used in such caase): when spinning up a cloud-like test server that communicates with a test instance of customer web server.
* `CWS_URL` (mandatory when `SERVER=cloud`, only used in such case): when spinning up a cloud-like test server that communicates with a test instance of customer web server.
* `TEST`: either `cypress` (default), `playwright`, or `none` (to avoid creating the cypress/playwright sidecar containers, e.g. if you only want to launch a server instance)
* `ENABLED_DOCKER_SERVICES`: a space-separated list of services to start alongside the server. Default to `postgres inbucket`, for smoke tests purposes and for lightweight and faster start up time. Depending on the test requirement being worked on, you may want to override as needed, as such:
- Cypress full tests require all services to be running: `postgres inbucket minio openldap elasticsearch keycloak`.
@ -29,6 +29,7 @@ Instructions, detailed:
* The `TEST_FILTER` variable can also be set, to customize which tests you want cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format.
2. (optional) `make start-dashboard && make generate-test-cycle`: start the automation-dashboard in the background, and initiate a test cycle on it, for the given `BUILD_ID`
* NB: the `BUILD_ID` value should stay the same across the `make generate-test-cycle` command, and the subsequent `make` (see next step). If you need to initiate a new test cycle on the same dashboard, you'll need to change the `BUILD_ID` value and rerun both `make generate-test-cycle` and `make`.
* Note that part of the dashboard functionality assumes the `BUILD_ID` to have a certain format (see [here](https://github.com/saturninoabril/automation-dashboard/blob/175891781bf1072c162c58c6ec0abfc5bcb3520e/lib/common_utils.ts#L3-L23) for details). This is not relevant for local running, but it's important to note in the testing pipelines.
* This also automatically sets the `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` variables for the cypress container
* Note that if you run the dashboard locally, but also specify other `AUTOMATION_DASHBOARD_*` variables in your `.ci/env` file, the latter variables will take precedence.
* The dashboard is used for orchestrating specs with parallel test run and is typically used in CI.
@ -40,7 +41,7 @@ Instructions, detailed:
* If you just want to run a local server instance, without any further testing, you can run `TEST=none make`
* If you're using the automation dashboard, you have the option of sharding the E2E test run: you can launch the `make` command in parallel on different machiness (NB: you must use the same `BUILD_ID` and `BRANCH` values that you used for `make generate-test-cycle`) to distribute running the test cases across them. When doing this, you should also set on each machine the `CI_BASE_URL` variable to a value that uniquely identifies the instance where `make` is running.
4. `make stop`: tears down the server (and the dashboard, if running)
* `make clean` will also remove any generated environment or docker-compose files, in addition to stopping the containers.
* This also implicitly runs `make clean`, which also remove any generated environment or docker-compose files.
Notes:
- Setting a variable in `.ci/env` is functionally equivalent to exporting variables in your current shell's environment, before invoking the makefile.