CLD-6867 E2E Tests - Decouple cycle creation from test running (#25367)

* Separate test cycle generation from test running

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

e2e-tests/.ci/.e2erc

@@ -16,6 +16,7 @@ export BROWSER_DEFAULT="chrome"
 export SERVER_DEFAULT="self-hosted"
 export TEST_DEFAULT="cypress"
 export ENABLED_DOCKER_SERVICES_DEFAULT="postgres inbucket"
+export TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
 # OS specific defaults overrides
 case $MME2E_OSTYPE in
   darwin )
@@ -29,6 +30,7 @@ export BROWSER=${BROWSER:-$BROWSER_DEFAULT}
 export SERVER=${SERVER:-$SERVER_DEFAULT}
 export TEST=${TEST:-$TEST_DEFAULT}
 export ENABLED_DOCKER_SERVICES=${ENABLED_DOCKER_SERVICES:-$ENABLED_DOCKER_SERVICES_DEFAULT}
+export TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT}
 
 # Function definitions
 mme2e_log () { echo "[$(date +%Y-%m-%dT%H:%M:%S%Z)]" "$@"; }

e2e-tests/.ci/dashboard.generate_test_cycle.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e -u -o pipefail
+cd "$(dirname "$0")"
+. .e2erc
+
+mme2e_log "Loading variables from .env.cypress"
+if ! [ -f .env.cypress ]; then
+  mme2e_log "Error: .env.cypress is required to exist, for the test cycle to be generated. Aborting." >&2
+  exit 1
+fi
+set -a
+. .env.cypress
+
+mme2e_log "Generating the test cycle on the Automation Dashboard"
+cd ../cypress
+# shellcheck disable=SC2086
+exec node --trace-warnings generate_test_cycle.js $TEST_FILTER

e2e-tests/.ci/server.generate.sh

@@ -172,9 +172,6 @@ $(if mme2e_is_token_in_list "cypress" "$ENABLED_DOCKER_SERVICES"; then
     env_file:
       - "../../e2e-tests/.ci/.env.cypress"
     environment:
-      REPO: "mattermost"
-      # Cypress configuration
-      HEADLESS: "true"
       CYPRESS_baseUrl: "http://localhost:8065"
       CYPRESS_dbConnection: "postgres://mmuser:mostest@localhost:5432/mattermost_test?sslmode=disable&connect_timeout=10"
       CYPRESS_smtpUrl: "http://localhost:9001"
@@ -290,6 +287,8 @@ generate_env_files() {
   export BRANCH=${BRANCH:-$BRANCH_DEFAULT}
   export BUILD_ID=${BUILD_ID:-$BUILD_ID_DEFAULT}
   export CI_BASE_URL="${CI_BASE_URL:-localhost}"
+  export REPO=mattermost # Static, but declared here for making generate_test_cycle.js easier to run
+  export HEADLESS=true   # Static, but declared here for making generate_test_cycle.js easier to run
   case "$TEST" in
   cypress)
     mme2e_log "Cypress: Generating .env.cypress"
@@ -298,8 +297,8 @@ generate_env_files() {
 	BUILD_ID
 	CI_BASE_URL
 	BROWSER
-	AUTOMATION_DASHBOARD_URL
-	AUTOMATION_DASHBOARD_TOKEN
+        HEADLESS
+        REPO
 	EOF
     # Adding service-specific cypress variables
     for SERVICE in $ENABLED_DOCKER_SERVICES; do

e2e-tests/.ci/server.run_cypress.sh

@@ -3,10 +3,6 @@ set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
 
-# Set required variables
-TEST_FILTER_DEFAULT='--stage=@prod --group=@smoke'
-TEST_FILTER=${TEST_FILTER:-$TEST_FILTER_DEFAULT}
-
 # Print run information
 mme2e_log "Printing Cypress container informations"
 ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node -p 'module.paths'
@@ -36,8 +32,6 @@ EOF
 # shellcheck disable=SC2016
 if ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress bash -c '[ -n "${AUTOMATION_DASHBOARD_URL}" ]'; then
   mme2e_log "AUTOMATION_DASHBOARD_URL is set. Using run_test_cycle.js for the cypress run"
-  # shellcheck disable=SC2086
-  ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node --trace-warnings generate_test_cycle.js $TEST_FILTER
   ${MME2E_DC_SERVER} exec -T -u "$MME2E_UID" -- cypress node run_test_cycle.js | tee ../cypress/logs/cypress.log
 else
   mme2e_log "AUTOMATION_DASHBOARD_URL is unset. Using run_tests.js for the cypress run"

e2e-tests/.ci/server.start.sh

@@ -3,12 +3,6 @@ set -e -u -o pipefail
 cd "$(dirname "$0")"
 . .e2erc
 
-BRANCH_DEFAULT=$(git branch --show-current)
-export BRANCH=${BRANCH:-$BRANCH_DEFAULT}
-BUILD_ID_DEFAULT=$(date +%s)
-export BUILD_ID=${BUILD_ID:-$BUILD_ID_DEFAULT}
-export CI_BASE_URL="${CI_BASE_URL:-localhost}"
-
 # Cleanup old containers, if any
 mme2e_log "Stopping leftover E2E containers, if any are running"
 ${MME2E_DC_SERVER} down -v --remove-orphans

e2e-tests/Makefile

@@ -25,9 +25,11 @@ stop-server: generate-server
 	bash ./.ci/server.stop.sh
 restart-server: stop-server start-server
 
-.PHONY: start-dashboard stop-dashboard
+.PHONY: start-dashboard generate-test-cycle stop-dashboard
 start-dashboard:
 	bash ./.ci/dashboard.start.sh
+generate-test-cycle: requirecmd-node generate-server
+	bash ./.ci/dashboard.generate_test_cycle.sh
 stop-dashboard:
 	bash ./.ci/dashboard.stop.sh
 

e2e-tests/README.md

@@ -10,7 +10,7 @@ Please refer to the [dedicated developer documentation](https://developers.matte
 
 ##### For pipeline debugging
 
-The E2E testing pipeline's scripts depend on the following tools being installed on your system: `docker`, `docker-compose`, `make`, `git`, `jq`, and some common utilities (`coreutils`, `findutils`, `bash`, `awk`, `sed`, `grep`)
+The E2E testing pipeline's scripts depend on the following tools being installed on your system: `docker`, `docker-compose`, `make`, `git`, `jq`, `node` and some common utilities (`coreutils`, `findutils`, `bash`, `awk`, `sed`, `grep`)
 
 Instructions, tl;dr: create a local branch with your E2E test changes, then open a PR to the `mattermost-server` repo targeting the `master` branch (so that CI will produce the image that docker-compose needs), then run `make` in this directory.
 
@@ -27,8 +27,9 @@ Instructions, detailed:
   * The following variables, which will be passed over to the cypress container: `BRANCH`, `BUILD_ID`, `CI_BASE_URL`, `BROWSER`, `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN`
   * The `SERVER_IMAGE` variable can also be set, if you want to select a custom mattermost-server image. If not specified, the value of the `SERVER_IMAGE_DEFAULT` variable defined in file `.ci/.e2erc` is used.
   * The `TEST_FILTER` variable can also be set, to customize which tests you want cypress to run. If not specified, only the smoke tests will run. Please check the `e2e-tests/cypress/run_tests.js` file for details about its format.
-2. (optional) `make start-dashboard`: start the automation-dashboard in the background
-  * This also sets the `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` variables for the cypress container
+2. (optional) `make start-dashboard && make generate-test-cycle`: start the automation-dashboard in the background, and initiate a test cycle on it, for the given `BUILD_ID`
+  * NB: the `BUILD_ID` value should stay the same across the `make generate-test-cycle` command, and the subsequent `make` (see next step). If you need to initiate a new test cycle on the same dashboard, you'll need to change the `BUILD_ID` value and rerun both `make generate-test-cycle` and `make`.
+  * This also automatically sets the `AUTOMATION_DASHBOARD_URL` and `AUTOMATION_DASHBOARD_TOKEN` variables for the cypress container
   * Note that if you run the dashboard locally, but also specify other `AUTOMATION_DASHBOARD_*` variables in your `.ci/env` file, the latter variables will take precedence.
   * The dashboard is used for orchestrating specs with parallel test run and is typically used in CI.
   * Only Cypress is currently using the dashboard; Playwright is not.
@@ -43,11 +44,12 @@ Instructions, detailed:
 Notes:
 - Setting a variable in `.ci/env` is functionally equivalent to exporting variables in your current shell's environment, before invoking the makefile.
 - The `.ci/.env.*` files are auto generated by the pipeline scripts, and aren't meant to be modified manually. The only file you should edit to control the containers' environment is `.ci/env`, as specified in the instructions above.
-- Aside from some exceptions (e.g. `TEST_FILTER`), most of the variables in `.ci/env` must be set before the `make generate-server` command is run. Modifying that file afterwards has no effect, because the containers' env files are generated in that step.
+- All of the variables in `.ci/env` must be set before the `make generate-server` command is run (or, if using the dashboard, before the `make generate-test-cycle` command). Modifying that file afterwards has no effect, because the containers' env files are generated in that step.
 - If you restart the dashboard at any point, you must also restart the server containers, so that it picks up the new IP of the dashboard from the newly generated `.env.dashboard` file
-- If new variables need to be passed to any of the containers:
+- If new variables need to be passed to any of the containers, here are the general principles to follow when deciding where to populate it:
   * If their value is fixed (e.g. a static server configuration), these may be simply added to the `docker_compose_generator.sh` file, to the appropriate container.
   * If you need to introduce variables that you want to control from `.ci/env`: you need to update the scripts under the `.ci/` dir, and configure them to write the new variables' values over to the appropriate `.env.*` file. In particular, avoid defining variables that depend on other variables within the docker-compose override files: this is to ensure uniformity in their availability, and simplifies the question of what container has access to which variable considerably.
+  * Exceptions are of course accepted wherever it makes sense (e.g. if you need to group variables based on some common functionality)
 
 ##### For code changes:
 * `make fmt-ci` to format and check yaml files and shell scripts.

e2e-tests/cypress/generate_test_cycle.js

@@ -44,7 +44,6 @@
  *   BROWSER                    : Chrome by default. Set to run test on other browser such as chrome, edge, electron and firefox.
  *                                The environment should have the specified browser to successfully run.
  *   HEADLESS                   : Headless by default (true) or false to run on headed mode.
- *   CI_BASE_URL                : Test server base URL in CI
  *
  * Example:
  * 1. "node generate_test_cycle.js"