SEC-9513 feat: e2e tests on master and releases (#35205)

* feat: e2e tests on master and releases

* (for pipelines testing only, will be removed after)

* remove test pipelines
This commit is contained in:
sabril
2026-02-11 13:02:25 +08:00
committed by GitHub
parent 1ac14a9dfb
commit a711b22717
16 changed files with 891 additions and 409 deletions
@@ -19131,8 +19131,10 @@ function calculateResultsFromSpecs(specs) {
const total = passed + failed;
const passRate = total > 0 ? (passed * 100 / total).toFixed(2) : "0.00";
const color = getColor(parseFloat(passRate));
const specSuffix = totalSpecs > 0 ? ` in ${totalSpecs} spec files` : "";
const commitStatusMessage = failed === 0 ? `${passed} passed${specSuffix}` : `${failed} failed, ${passed} passed${specSuffix}`;
const rate = total > 0 ? passed * 100 / total : 0;
const rateStr = rate === 100 ? "100%" : `${rate.toFixed(1)}%`;
const specSuffix = totalSpecs > 0 ? `, ${totalSpecs} specs` : "";
const commitStatusMessage = rate === 100 ? `${rateStr} passed (${passed})${specSuffix}` : `${rateStr} passed (${passed}/${total}), ${failed} failed${specSuffix}`;
return {
passed,
failed,
@@ -108,7 +108,7 @@ describe("calculateResultsFromSpecs", () => {
expect(calc.totalSpecs).toBe(2);
expect(calc.failedSpecs).toBe("");
expect(calc.failedSpecsCount).toBe(0);
expect(calc.commitStatusMessage).toBe("2 passed in 2 spec files");
expect(calc.commitStatusMessage).toBe("100% passed (2), 2 specs");
});
it("should calculate all outputs correctly for results with failures", () => {
@@ -136,7 +136,7 @@ describe("calculateResultsFromSpecs", () => {
expect(calc.failedSpecs).toBe("tests/integration/channels.spec.ts");
expect(calc.failedSpecsCount).toBe(1);
expect(calc.commitStatusMessage).toBe(
"1 failed, 1 passed in 2 spec files",
"50.0% passed (1/2), 1 failed, 2 specs",
);
expect(calc.failedTests).toContain("should create a channel");
});
@@ -230,7 +230,7 @@ describe("merge simulation", () => {
expect(finalCalc.totalSpecs).toBe(3);
expect(finalCalc.failedSpecs).toBe("");
expect(finalCalc.failedSpecsCount).toBe(0);
expect(finalCalc.commitStatusMessage).toBe("3 passed in 3 spec files");
expect(finalCalc.commitStatusMessage).toBe("100% passed (3), 3 specs");
});
it("should handle case where retest still fails", () => {
@@ -165,11 +165,13 @@ export function calculateResultsFromSpecs(
const color = getColor(parseFloat(passRate));
// Build commit status message
const specSuffix = totalSpecs > 0 ? ` in ${totalSpecs} spec files` : "";
const rate = total > 0 ? (passed * 100) / total : 0;
const rateStr = rate === 100 ? "100%" : `${rate.toFixed(1)}%`;
const specSuffix = totalSpecs > 0 ? `, ${totalSpecs} specs` : "";
const commitStatusMessage =
failed === 0
? `${passed} passed${specSuffix}`
: `${failed} failed, ${passed} passed${specSuffix}`;
rate === 100
? `${rateStr} passed (${passed})${specSuffix}`
: `${rateStr} passed (${passed}/${total}), ${failed} failed${specSuffix}`;
return {
passed,
@@ -19173,8 +19173,10 @@ function calculateResults(results) {
const total = passing + failed;
const passRate = total > 0 ? (passing * 100 / total).toFixed(2) : "0.00";
const color = getColor(parseFloat(passRate));
const specSuffix = totalSpecs > 0 ? ` in ${totalSpecs} spec files` : "";
const commitStatusMessage = failed === 0 ? `${passed} passed${specSuffix}` : `${failed} failed, ${passed} passed${specSuffix}`;
const rate = total > 0 ? passing * 100 / total : 0;
const rateStr = rate === 100 ? "100%" : `${rate.toFixed(1)}%`;
const specSuffix = totalSpecs > 0 ? `, ${totalSpecs} specs` : "";
const commitStatusMessage = rate === 100 ? `${rateStr} passed (${passing})${specSuffix}` : `${rateStr} passed (${passing}/${total}), ${failed} failed${specSuffix}`;
return {
passed,
failed,
@@ -262,7 +262,7 @@ describe("calculateResults", () => {
expect(calc.totalSpecs).toBe(2);
expect(calc.failedSpecs).toBe("");
expect(calc.failedSpecsCount).toBe(0);
expect(calc.commitStatusMessage).toBe("2 passed in 2 spec files");
expect(calc.commitStatusMessage).toBe("100% passed (2), 2 specs");
});
it("should calculate all outputs correctly for results with failures", () => {
@@ -305,7 +305,7 @@ describe("calculateResults", () => {
expect(calc.failedSpecs).toBe("channels.spec.ts");
expect(calc.failedSpecsCount).toBe(1);
expect(calc.commitStatusMessage).toBe(
"1 failed, 1 passed in 2 spec files",
"50.0% passed (1/2), 1 failed, 2 specs",
);
expect(calc.failedTests).toContain("should create channel");
});
@@ -445,7 +445,7 @@ describe("full integration: original with failure, retest passes", () => {
expect(finalCalc.totalSpecs).toBe(3);
expect(finalCalc.failedSpecs).toBe("");
expect(finalCalc.failedSpecsCount).toBe(0);
expect(finalCalc.commitStatusMessage).toBe("3 passed in 3 spec files");
expect(finalCalc.commitStatusMessage).toBe("100% passed (3), 3 specs");
expect(finalCalc.failedTests).toBe("");
});
@@ -228,11 +228,13 @@ export function calculateResults(
const color = getColor(parseFloat(passRate));
// Build commit status message
const specSuffix = totalSpecs > 0 ? ` in ${totalSpecs} spec files` : "";
const rate = total > 0 ? (passing * 100) / total : 0;
const rateStr = rate === 100 ? "100%" : `${rate.toFixed(1)}%`;
const specSuffix = totalSpecs > 0 ? `, ${totalSpecs} specs` : "";
const commitStatusMessage =
failed === 0
? `${passed} passed${specSuffix}`
: `${failed} failed, ${passed} passed${specSuffix}`;
rate === 100
? `${rateStr} passed (${passing})${specSuffix}`
: `${rateStr} passed (${passing}/${total}), ${failed} failed${specSuffix}`;
return {
passed,
-320
View File
@@ -1,320 +0,0 @@
# E2E Test Workflow For PR
This document describes the E2E test workflow for Pull Requests in Mattermost.
## Overview
This is an **automated workflow** that runs smoke-then-full E2E tests automatically for every PR commit. Smoke tests run first as a gate—if they fail, full tests are skipped to save CI resources and provide fast feedback.
Both Cypress and Playwright test suites run **in parallel** with independent status checks.
**Note**: This workflow is designed for **Pull Requests only**. It will fail if the commit SHA is not associated with an open PR.
### On-Demand Testing
For on-demand E2E testing, the existing triggers still work:
- **Comment triggers**: `/e2e-test`, `/e2e-test fips`, or with `MM_ENV` parameters
- **Label trigger**: `E2E/Run`
These manual triggers are separate from this automated workflow and can be used for custom test configurations or re-runs.
## Workflow Files
```
.github/workflows/
├── e2e-tests-ci.yml # Main orchestrator (resolves PR, triggers both)
├── e2e-tests-cypress.yml # Cypress: smoke → full
└── e2e-tests-playwright.yml # Playwright: smoke → full
```
## Architecture Diagram
```
┌─────────────────────────────────────────────────────────────────────────────────┐
│ MAIN ORCHESTRATOR: e2e-tests-ci.yml │
└─────────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────┐
│ workflow_dispatch │
│ (commit_sha) │
└──────────┬──────────┘
┌──────────▼──────────┐
│ resolve-pr │
│ (GitHub API call) │
│ │
│ Fails if no PR │
│ found for commit │
└──────────┬──────────┘
┌──────────────────┴──────────────────┐
│ (parallel) │
▼ ▼
┌─────────────────────────────────┐ ┌─────────────────────────────────┐
│ e2e-tests-cypress.yml │ │ e2e-tests-playwright.yml │
│ (reusable workflow) │ │ (reusable workflow) │
│ │ │ │
│ Inputs: │ │ Inputs: │
│ • commit_sha │ │ • commit_sha │
│ • workers_number: "20" │ │ • workers_number: "1" (default)│
│ • server: "onprem" │ │ • server: "onprem" │
│ • enable_reporting: true │ │ • enable_reporting: true │
│ • report_type: "PR" │ │ • report_type: "PR" │
│ • pr_number │ │ • pr_number (required for full)│
└─────────────────────────────────┘ └─────────────────────────────────┘
```
## Per-Framework Workflow Flow
Each framework (Cypress/Playwright) follows the same pattern:
```
┌──────────────────────────────────────────────────────────────────┐
│ PREFLIGHT CHECKS │
└──────────────────────────────────────────────────────────────────┘
┌─────────────────────────┼─────────────────────────┐
│ │ │
▼ ▼ ▼
┌────────────┐ ┌─────────────┐ ┌─────────────┐
│ lint/tsc │ │ shell-check │ │ update- │
│ check │ │ │ │ status │
└─────┬──────┘ └──────┬──────┘ │ (pending) │
│ │ └──────┬──────┘
└──────────────────────┴────────────────────────┘
┌──────────────────────────────────────────────────────────────────┐
│ GENERATE BUILD VARIABLES │
│ (branch, build_id, server_image) │
│ │
│ Server image generated from commit SHA: │
│ mattermostdevelopment/mattermost-enterprise-edition:<sha7> │
└─────────────────────────────┬────────────────────────────────────┘
┌──────────────────────────────────────────────────────────────────┐
│ SMOKE TESTS │
│ ┌────────────────────────────────────────────────────────────┐ │
│ │ generate-test-cycle (smoke) [Cypress only] │ │
│ └─────────────────────────┬──────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌────────────────────────────────────────────────────────────┐ │
│ │ smoke-test │ │
│ │ • Cypress: TEST_FILTER: --stage=@prod --group=@smoke │ │
│ │ • Playwright: TEST_FILTER: --grep @smoke │ │
│ │ • Fail fast if any smoke test fails │ │
│ └─────────────────────────┬──────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌────────────────────────────────────────────────────────────┐ │
│ │ smoke-report │ │
│ │ • Assert 0 failures │ │
│ │ • Upload results to S3 (Playwright) │ │
│ │ • Update commit status │ │
│ └────────────────────────────────────────────────────────────┘ │
└─────────────────────────────┬────────────────────────────────────┘
│ (only if smoke passes)
│ (Playwright: also requires pr_number)
┌──────────────────────────────────────────────────────────────────┐
│ FULL TESTS │
│ ┌────────────────────────────────────────────────────────────┐ │
│ │ generate-test-cycle (full) [Cypress only] │ │
│ └─────────────────────────┬──────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌────────────────────────────────────────────────────────────┐ │
│ │ full-test (matrix: workers) │ │
│ │ • Cypress: TEST_FILTER: --stage='@prod' │ │
│ │ --exclude-group='@smoke' │ │
│ │ • Playwright: TEST_FILTER: --grep-invert "@smoke|@visual" │ │
│ │ • Multiple workers for parallelism │ │
│ └─────────────────────────┬──────────────────────────────────┘ │
│ │ │
│ ▼ │
│ ┌────────────────────────────────────────────────────────────┐ │
│ │ full-report │ │
│ │ • Aggregate results from all workers │ │
│ │ • Upload results to S3 (Playwright) │ │
│ │ • Publish report (if reporting enabled) │ │
│ │ • Update final commit status │ │
│ └────────────────────────────────────────────────────────────┘ │
└──────────────────────────────────────────────────────────────────┘
```
## Commit Status Checks
Each workflow phase creates its own GitHub commit status check:
```
GitHub Commit Status Checks:
═══════════════════════════
┌─────────────────────────────────────────────────────────────────────────────┐
│ E2E Tests/cypress-smoke ●────────●────────● │
│ pending running ✓ passed / ✗ failed │
│ │
│ E2E Tests/cypress-full ○ ○ ●────────●────────● │
│ (skip) (skip) pending running ✓/✗ │
│ │ │
│ └── Only runs if smoke passes │
└─────────────────────────────────────────────────────────────────────────────┘
┌─────────────────────────────────────────────────────────────────────────────┐
│ E2E Tests/playwright-smoke ●────────●────────● │
│ pending running ✓ passed / ✗ failed │
│ │
│ E2E Tests/playwright-full ○ ○ ●────────●────────● │
│ (skip) (skip) pending running ✓/✗ │
│ │ │
│ └── Only runs if smoke passes │
│ AND pr_number is provided │
└─────────────────────────────────────────────────────────────────────────────┘
```
## Timeline
```
Timeline:
─────────────────────────────────────────────────────────────────────────────►
T0 T1 T2 T3 T4
│ │ │ │ │
│ Start │ Preflight │ Smoke Tests │ Full Tests │ Done
│ resolve │ Checks │ (both parallel) │ (both parallel) │
│ PR │ │ │ (if smoke pass) │
```
## Test Filtering
| Framework | Smoke Tests | Full Tests |
|-----------|-------------|------------|
| **Cypress** | `--stage=@prod --group=@smoke` | See below |
| **Playwright** | `--grep @smoke` | `--grep-invert "@smoke\|@visual"` |
### Cypress Full Test Filter
```
--stage="@prod"
--excludeGroup="@smoke,@te_only,@cloud_only,@high_availability"
--sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap"
--sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"
```
- **excludeGroup**: Skips smoke tests (already run), TE-only, cloud-only, and HA tests
- **sortFirst**: Runs long-running test groups early for better parallelization
- **sortLast**: Runs tests that may affect system state at the end
## Tagging Smoke Tests
### Cypress
Add `@smoke` to the Group comment at the top of spec files:
```javascript
// Stage: @prod
// Group: @channels @messaging @smoke
```
### Playwright
Add `@smoke` to the test tag option:
```typescript
test('critical login flow', {tag: ['@smoke', '@login']}, async ({pw}) => {
// ...
});
```
## Worker Configuration
| Framework | Smoke Workers | Full Workers |
|-----------|---------------|--------------|
| **Cypress** | 1 | 20 |
| **Playwright** | 1 | 1 (uses internal parallelism via `PW_WORKERS`) |
## Docker Services
Different test phases enable different Docker services based on test requirements:
| Test Phase | Docker Services |
|------------|-----------------|
| Smoke Tests | `postgres inbucket` |
| Full Tests | `postgres inbucket minio openldap elasticsearch keycloak` |
Full tests enable additional services to support tests requiring LDAP, Elasticsearch, S3-compatible storage (Minio), and SAML/OAuth (Keycloak).
## Failure Behavior
1. **Smoke test fails**: Full tests are skipped, only smoke commit status shows failure (no full test status created)
2. **Full test fails**: Full commit status shows failure with details
3. **Both pass**: Both smoke and full commit statuses show success
4. **No PR found**: Workflow fails immediately with error message
**Note**: Full test status updates use explicit job result checks (`needs.full-report.result == 'success'` / `'failure'`) rather than global `success()` / `failure()` functions. This ensures full test status is only updated when full tests actually run, not when smoke tests fail upstream.
## Manual Trigger
The workflow can be triggered manually via `workflow_dispatch` for PR commits:
```bash
# Run E2E tests for a PR commit
gh workflow run e2e-tests-ci.yml -f commit_sha=<PR_COMMIT_SHA>
```
**Note**: The commit SHA must be associated with an open PR. The workflow will fail otherwise.
## Automated Trigger (Argo Events)
The workflow is automatically triggered by Argo Events when the `Enterprise CI/docker-image` status check succeeds on a commit.
### Fork PR Handling
For PRs from forked repositories:
- `body.branches` may be empty (commit doesn't exist in base repo branches)
- Falls back to `master` branch for workflow files (trusted code)
- The `commit_sha` still points to the fork's commit for testing
- PR number is resolved via GitHub API (works for fork PRs)
### Flow
```
Enterprise CI/docker-image succeeds
Argo Events Sensor
workflow_dispatch
(ref, commit_sha)
e2e-tests-ci.yml
resolve-pr (GitHub API)
Cypress + Playwright (parallel)
```
## S3 Report Storage
Playwright test results are uploaded to S3:
| Test Phase | S3 Path |
|------------|---------|
| Smoke (with PR) | `server-pr-{PR_NUMBER}/e2e-reports/playwright-smoke/{RUN_ID}/` |
| Smoke (no PR) | `server-commit-{SHA7}/e2e-reports/playwright-smoke/{RUN_ID}/` |
| Full | `server-pr-{PR_NUMBER}/e2e-reports/playwright-full/{RUN_ID}/` |
**Note**: Full tests require a PR number, so there's no commit-based fallback for full test reports.
## Related Files
- `e2e-tests/cypress/` - Cypress test suite
- `e2e-tests/playwright/` - Playwright test suite
- `e2e-tests/.ci/` - CI configuration and environment files
- `e2e-tests/Makefile` - Main Makefile with targets for running tests, generating cycles, and reporting
+352
View File
@@ -0,0 +1,352 @@
# E2E Test Pipelines
Three automated E2E test pipelines cover different stages of the development lifecycle.
## Pipelines
| Pipeline | Trigger | Editions Tested | Image Source |
|----------|---------|----------------|--------------|
| **PR** (`e2e-tests-ci.yml`) | Argo Events on `Enterprise CI/docker-image` status | enterprise | `mattermostdevelopment/**` |
| **Merge to master/release** (`e2e-tests-on-merge.yml`) | Platform delivery after docker build (`delivery-platform/.github/workflows/mattermost-platform-delivery.yaml`) | enterprise, fips | `mattermostdevelopment/**` |
| **Release cut** (`e2e-tests-on-release.yml`) | Platform release after docker build (`delivery-platform/.github/workflows/release-mattermost-platform.yml`) | enterprise, fips, team (future) | `mattermost/**` |
All pipelines follow the **smoke-then-full** pattern: smoke tests run first, full tests only run if smoke passes.
## Workflow Files
```
.github/workflows/
├── e2e-tests-ci.yml # PR orchestrator
├── e2e-tests-on-merge.yml # Merge orchestrator (master/release branches)
├── e2e-tests-on-release.yml # Release cut orchestrator
├── e2e-tests-cypress.yml # Shared wrapper: cypress smoke -> full
├── e2e-tests-playwright.yml # Shared wrapper: playwright smoke -> full
├── e2e-tests-cypress-template.yml # Template: actual cypress test execution
└── e2e-tests-playwright-template.yml # Template: actual playwright test execution
```
### Call hierarchy
```
e2e-tests-ci.yml ─────────────────┐
e2e-tests-on-merge.yml ───────────┤──► e2e-tests-cypress.yml ──► e2e-tests-cypress-template.yml
e2e-tests-on-release.yml ─────────┘ e2e-tests-playwright.yml ──► e2e-tests-playwright-template.yml
```
---
## Pipeline 1: PR (`e2e-tests-ci.yml`)
Runs E2E tests for every PR commit after the enterprise docker image is built. Fails if the commit is not associated with an open PR.
**Trigger chain:**
```
PR commit ─► Enterprise CI builds docker image
─► Argo Events detects "Enterprise CI/docker-image" status
─► dispatches e2e-tests-ci.yml
```
For PRs from forks, `body.branches` may be empty so the workflow falls back to `master` for workflow files (trusted code), while `commit_sha` still points to the fork's commit.
**Jobs:** 2 (cypress + playwright), each does smoke -> full
**Commit statuses (4 total):**
| Context | Description (pending) | Description (result) |
|---------|----------------------|---------------------|
| `e2e-test/cypress-smoke\|enterprise` | `tests running, image_tag:abc1234` | `100% passed (1313), 440 specs, image_tag:abc1234` |
| `e2e-test/cypress-full\|enterprise` | `tests running, image_tag:abc1234` | `100% passed (1313), 440 specs, image_tag:abc1234` |
| `e2e-test/playwright-smoke\|enterprise` | `tests running, image_tag:abc1234` | `100% passed (200), 50 specs, image_tag:abc1234` |
| `e2e-test/playwright-full\|enterprise` | `tests running, image_tag:abc1234` | `99.5% passed (199/200), 1 failed, 50 specs, image_tag:abc1234` |
**Manual trigger (CLI):**
```bash
gh workflow run e2e-tests-ci.yml \
--repo mattermost/mattermost \
--field pr_number="35171"
```
**Manual trigger (GitHub UI):**
1. Go to **Actions** > **E2E Tests (smoke-then-full)**
2. Click **Run workflow**
3. Fill in `pr_number` (e.g., `35171`)
4. Click **Run workflow**
### On-demand testing
For on-demand E2E testing, the existing triggers still work:
- **Comment triggers**: `/e2e-test`, `/e2e-test fips`, or with `MM_ENV` parameters
- **Label trigger**: `E2E/Run`
These are separate from the automated workflow and can be used for custom test configurations or re-runs.
---
## Pipeline 2: Merge (`e2e-tests-on-merge.yml`)
Runs E2E tests after every push/merge to `master` or `release-*` branches.
**Trigger chain:**
```
Push to master/release-*
─► Argo Events (mattermost-platform-package sensor)
─► delivery-platform/.github/workflows/mattermost-platform-delivery.yaml
─► builds docker images (enterprise + fips)
─► trigger-e2e-tests job dispatches e2e-tests-on-merge.yml
```
**Jobs:** 4 (cypress + playwright) x (enterprise + fips), smoke skipped, full tests only
**Commit statuses (4 total):**
| Context | Description example |
|---------|-------------------|
| `e2e-test/cypress-full\|enterprise` | `100% passed (1313), 440 specs, image_tag:abc1234_def5678` |
| `e2e-test/cypress-full\|fips` | `100% passed (1313), 440 specs, image_tag:abc1234_def5678` |
| `e2e-test/playwright-full\|enterprise` | `100% passed (200), 50 specs, image_tag:abc1234_def5678` |
| `e2e-test/playwright-full\|fips` | `100% passed (200), 50 specs, image_tag:abc1234_def5678` |
**Manual trigger (CLI):**
```bash
# For master
gh workflow run e2e-tests-on-merge.yml \
--repo mattermost/mattermost \
--field branch="master" \
--field commit_sha="<full_commit_sha>" \
--field server_image_tag="<image_tag>"
# For release branch
gh workflow run e2e-tests-on-merge.yml \
--repo mattermost/mattermost \
--field branch="release-11.4" \
--field commit_sha="<full_commit_sha>" \
--field server_image_tag="<image_tag>"
```
**Manual trigger (GitHub UI):**
1. Go to **Actions** > **E2E Tests (master/release - merge)**
2. Click **Run workflow**
3. Fill in:
- `branch`: `master` or `release-11.4`
- `commit_sha`: full 40-char SHA
- `server_image_tag`: e.g., `abc1234_def5678`
4. Click **Run workflow**
---
## Pipeline 3: Release Cut (`e2e-tests-on-release.yml`)
Runs E2E tests after a release cut against the published release images.
**Trigger chain:**
```
Manual release cut
─► delivery-platform/.github/workflows/release-mattermost-platform.yml
─► builds and publishes release docker images
─► trigger-e2e-tests job dispatches e2e-tests-on-release.yml
```
**Jobs:** 4 (cypress + playwright) x (enterprise + fips), smoke skipped, full tests only. Team edition planned for future.
**Commit statuses (4 total, 6 when team is enabled):**
Descriptions include alias tags showing which rolling docker tags point to the same image.
RC example (11.4.0-rc3):
| Context | Description example |
|---------|-------------------|
| `e2e-test/cypress-full\|enterprise` | `100% passed (1313), 440 specs, image_tag:11.4.0-rc3 (release-11.4, release-11)` |
| `e2e-test/cypress-full\|fips` | `100% passed (1313), 440 specs, image_tag:11.4.0-rc3 (release-11.4, release-11)` |
| `e2e-test/cypress-full\|team` (future) | `100% passed (1313), 440 specs, image_tag:11.4.0-rc3 (release-11.4, release-11)` |
Stable example (11.4.0) — includes `MAJOR.MINOR` alias:
| Context | Description example |
|---------|-------------------|
| `e2e-test/cypress-full\|enterprise` | `100% passed (1313), 440 specs, image_tag:11.4.0 (release-11.4, release-11, 11.4)` |
| `e2e-test/cypress-full\|fips` | `100% passed (1313), 440 specs, image_tag:11.4.0 (release-11.4, release-11, 11.4)` |
| `e2e-test/cypress-full\|team` (future) | `100% passed (1313), 440 specs, image_tag:11.4.0 (release-11.4, release-11, 11.4)` |
**Manual trigger (CLI):**
```bash
gh workflow run e2e-tests-on-release.yml \
--repo mattermost/mattermost \
--field branch="release-11.4" \
--field commit_sha="<full_commit_sha>" \
--field server_image_tag="11.4.0" \
--field server_image_aliases="release-11.4, release-11, 11.4"
```
**Manual trigger (GitHub UI):**
1. Go to **Actions** > **E2E Tests (release cut)**
2. Click **Run workflow**
3. Fill in:
- `branch`: `release-11.4`
- `commit_sha`: full 40-char SHA
- `server_image_tag`: e.g., `11.4.0` or `11.4.0-rc3`
- `server_image_aliases`: e.g., `release-11.4, release-11, 11.4` (optional)
4. Click **Run workflow**
---
## Commit Status Format
**Context name:** `e2e-test/<phase>|<edition>`
Where `<phase>` is `cypress-smoke`, `cypress-full`, `playwright-smoke`, or `playwright-full`.
**Description format:**
- All passed: `100% passed (<count>), <specs> specs, image_tag:<tag>[ (<aliases>)]`
- With failures: `<rate>% passed (<passed>/<total>), <failed> failed, <specs> specs, image_tag:<tag>[ (<aliases>)]`
- Pending: `tests running, image_tag:<tag>[ (<aliases>)]`
- Pass rate: `100%` if all pass, otherwise one decimal (e.g., `99.5%`)
- Aliases only present for release cuts
### Failure behavior
1. **Smoke test fails**: Full tests are skipped, only smoke commit status shows failure
2. **Full test fails**: Full commit status shows failure with pass rate
3. **Both pass**: Both smoke and full commit statuses show success
4. **No PR found** (PR pipeline only): Workflow fails immediately
---
## Smoke-then-Full Pattern
Each wrapper (Cypress/Playwright) follows this flow:
```
generate-build-variables (branch, build_id, server_image)
─► smoke tests (1 worker, minimal docker services)
─► if smoke passes ─► full tests (20 workers cypress / 1 worker playwright, all docker services)
─► report (aggregate results, update commit status)
```
### Test filtering
| Framework | Smoke | Full |
|-----------|-------|------|
| **Cypress** | `--stage=@prod --group=@smoke` | `--stage="@prod" --excludeGroup="@te_only,@cloud_only,@high_availability" --sortFirst=... --sortLast=...` |
| **Playwright** | `--grep @smoke` | `--grep-invert "@smoke\|@visual"` |
### Worker configuration
| Framework | Smoke Workers | Full Workers |
|-----------|---------------|--------------|
| **Cypress** | 1 | 20 |
| **Playwright** | 1 | 1 (uses internal parallelism via `PW_WORKERS`) |
### Docker services
| Test Phase | Docker Services |
|------------|-----------------|
| Smoke | `postgres inbucket` |
| Full | `postgres inbucket minio openldap elasticsearch keycloak` |
---
## Tagging Smoke Tests
### Cypress
Add `@smoke` to the Group comment at the top of spec files:
```javascript
// Stage: @prod
// Group: @channels @messaging @smoke
```
### Playwright
Add `@smoke` to the test tag option:
```typescript
test('critical login flow', {tag: ['@smoke', '@login']}, async ({pw}) => {
// ...
});
```
---
## Shared Wrapper Inputs
The wrappers (`e2e-tests-cypress.yml`, `e2e-tests-playwright.yml`) accept these inputs:
| Input | Default | Description |
|-------|---------|-------------|
| `server_edition` | `enterprise` | Edition: `enterprise`, `fips`, or `team` |
| `server_image_repo` | `mattermostdevelopment` | Docker namespace: `mattermostdevelopment` or `mattermost` |
| `server_image_tag` | derived from `commit_sha` | Docker image tag |
| `server_image_aliases` | _(empty)_ | Alias tags shown in commit status description |
| `ref_branch` | _(empty)_ | Source branch name for webhook messages (e.g., `master` or `release-11.4`) |
The automation dashboard branch name is derived from context:
- PR: `server-pr-<pr_number>` (e.g., `server-pr-35205`)
- Master merge: `server-master-<image_tag>` (e.g., `server-master-abc1234_def5678`)
- Release merge: `server-release-<version>-<image_tag>` (e.g., `server-release-11.4-abc1234_def5678`)
- Fallback: `server-commit-<image_tag>`
The test type suffix (`-smoke` or `-full`) is appended by the template.
The server image is derived as:
```
{server_image_repo}/{edition_image_name}:{server_image_tag}
```
Where `edition_image_name` maps to:
- `enterprise` -> `mattermost-enterprise-edition`
- `fips` -> `mattermost-enterprise-fips-edition`
- `team` -> `mattermost-team-edition`
---
## Webhook Message Format
After full tests complete, a webhook notification is sent to the configured `REPORT_WEBHOOK_URL`. The results line uses the same `commit_status_message` as the GitHub commit status. The source line varies by pipeline using `report_type` and `ref_branch`.
**Report types:** `PR`, `MASTER`, `RELEASE`, `RELEASE_CUT`
### PR
```
:open-pull-request: mattermost-pr-35205
:docker: mattermostdevelopment/mattermost-enterprise-edition:abc1234
100% passed (1313), 440 specs | full report
```
### Merge to master
```
:git_merge: abc1234 on master
:docker: mattermostdevelopment/mattermost-enterprise-edition:abc1234_def5678
100% passed (1313), 440 specs | full report
```
### Merge to release branch
```
:git_merge: abc1234 on release-11.4
:docker: mattermostdevelopment/mattermost-enterprise-edition:abc1234_def5678
100% passed (1313), 440 specs | full report
```
### Release cut
```
:github_round: abc1234 on release-11.4
:docker: mattermost/mattermost-enterprise-edition:11.4.0-rc3
100% passed (1313), 440 specs | full report
```
The commit short SHA links to the commit on GitHub. The PR number links to the pull request.
---
## Related Files
- `e2e-tests/cypress/` - Cypress test suite
- `e2e-tests/playwright/` - Playwright test suite
- `e2e-tests/.ci/` - CI configuration and environment files
- `e2e-tests/Makefile` - Makefile with targets for running tests, generating cycles, and reporting
@@ -46,6 +46,20 @@ on:
type: string
required: false
default: onprem
server_edition:
description: "Server edition: enterprise (default), fips, or team"
type: string
required: false
default: enterprise
server_image_repo:
description: "Docker registry: mattermostdevelopment (default) or mattermost"
type: string
required: false
default: mattermostdevelopment
server_image_aliases:
description: "Comma-separated alias tags for description (e.g., 'release-11.4, release-11')"
type: string
required: false
# Reporting options
enable_reporting:
@@ -55,6 +69,10 @@ on:
report_type:
type: string
required: false
ref_branch:
description: "Source branch name for webhook messages (e.g., 'master' or 'release-11.4')"
type: string
required: false
pr_number:
type: string
required: false
@@ -92,7 +110,7 @@ on:
required: false
env:
SERVER_IMAGE: "mattermostdevelopment/mattermost-enterprise-edition:${{ inputs.server_image_tag }}"
SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}"
jobs:
update-initial-status:
@@ -106,7 +124,7 @@ jobs:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "with image tag: ${{ inputs.server_image_tag }}"
description: "tests running, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: pending
generate-test-cycle:
@@ -121,12 +139,12 @@ jobs:
echo "workers=$(jq -nc '[range(${{ inputs.workers }})]')" >> $GITHUB_OUTPUT
- name: ci/checkout-repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: ".nvmrc"
cache: npm
@@ -187,7 +205,7 @@ jobs:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: ".nvmrc"
cache: npm
@@ -203,7 +221,7 @@ jobs:
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: cypress-${{ inputs.test_type }}-results-${{ matrix.worker_index }}
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-${{ matrix.worker_index }}
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
@@ -229,14 +247,11 @@ jobs:
color: ${{ steps.calculate.outputs.color }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: ci/download-results
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
pattern: cypress-${{ inputs.test_type }}-results-*
pattern: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-*
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/calculate
@@ -279,7 +294,7 @@ jobs:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: ".nvmrc"
cache: npm
@@ -298,7 +313,7 @@ jobs:
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: cypress-${{ inputs.test_type }}-retest-results
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-retest-results
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
@@ -321,12 +336,9 @@ jobs:
working-directory: e2e-tests
steps:
- name: ci/checkout-repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: ci/setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: ".nvmrc"
cache: npm
@@ -337,7 +349,7 @@ jobs:
if: needs.run-failed-tests.result == 'skipped'
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
pattern: cypress-${{ inputs.test_type }}-results-*
pattern: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-*
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/use-previous-calculation
@@ -365,14 +377,14 @@ jobs:
if: needs.run-failed-tests.result != 'skipped'
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
pattern: cypress-${{ inputs.test_type }}-results-*
pattern: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results-*
path: e2e-tests/cypress/
merge-multiple: true
- name: ci/download-retest-results
if: needs.run-failed-tests.result != 'skipped'
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: cypress-${{ inputs.test_type }}-retest-results
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-retest-results
path: e2e-tests/cypress/retest-results/
- name: ci/calculate-results
if: needs.run-failed-tests.result != 'skipped'
@@ -427,7 +439,7 @@ jobs:
if: inputs.workers > 1
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
with:
name: cypress-${{ inputs.test_type }}-results
name: cypress-${{ inputs.test_type }}-${{ inputs.server_edition }}-results
path: |
e2e-tests/cypress/logs/
e2e-tests/cypress/results/
@@ -435,18 +447,29 @@ jobs:
if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != ''
env:
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
PASS_RATE: ${{ steps.final-results.outputs.pass_rate }}
PASSED: ${{ steps.final-results.outputs.passed }}
TOTAL: ${{ steps.final-results.outputs.total }}
TOTAL_SPECS: ${{ steps.final-results.outputs.total_specs }}
COMMIT_STATUS_MESSAGE: ${{ steps.final-results.outputs.commit_status_message }}
COLOR: ${{ steps.final-results.outputs.color }}
REPORT_URL: ${{ needs.generate-test-cycle.outputs.status_check_url }}
TEST_TYPE: ${{ inputs.test_type }}
REPORT_TYPE: ${{ inputs.report_type }}
COMMIT_SHA: ${{ inputs.commit_sha }}
REF_BRANCH: ${{ inputs.ref_branch }}
PR_NUMBER: ${{ inputs.pr_number }}
run: |
# Capitalize test type
TEST_TYPE_CAP=$(echo "$TEST_TYPE" | sed 's/.*/\u&/')
# Build source line based on report type
COMMIT_SHORT="${COMMIT_SHA::7}"
COMMIT_URL="https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}"
if [ "$REPORT_TYPE" = "RELEASE_CUT" ]; then
SOURCE_LINE=":github_round: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`"
elif [ "$REPORT_TYPE" = "MASTER" ] || [ "$REPORT_TYPE" = "RELEASE" ]; then
SOURCE_LINE=":git_merge: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`"
else
SOURCE_LINE=":open-pull-request: [mattermost-pr-${PR_NUMBER}](https://github.com/${{ github.repository }}/pull/${PR_NUMBER})"
fi
# Build payload with attachments
PAYLOAD=$(cat <<EOF
{
@@ -454,7 +477,7 @@ jobs:
"icon_url": "https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png",
"attachments": [{
"color": "${COLOR}",
"text": "**Results - Cypress ${TEST_TYPE_CAP} Tests**\n\n:open-pull-request: [mattermost-pr-${PR_NUMBER}](https://github.com/${{ github.repository }}/pull/${PR_NUMBER})\n:docker: \`${{ env.SERVER_IMAGE }}\`\n${PASS_RATE}% (${PASSED}/${TOTAL}) in ${TOTAL_SPECS} spec files | [full report](${REPORT_URL})"
"text": "**Results - Cypress ${TEST_TYPE_CAP} Tests**\n\n${SOURCE_LINE}\n:docker: \`${{ env.SERVER_IMAGE }}\`\n${COMMIT_STATUS_MESSAGE} | [full report](${REPORT_URL})"
}]
}
EOF
@@ -528,7 +551,7 @@ jobs:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "${{ needs.report.outputs.commit_status_message }} with image tag: ${{ inputs.server_image_tag }}"
description: "${{ needs.report.outputs.commit_status_message }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: success
target_url: ${{ needs.generate-test-cycle.outputs.status_check_url }}
@@ -547,6 +570,6 @@ jobs:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "${{ needs.report.outputs.commit_status_message }} with image tag: ${{ inputs.server_image_tag }}"
description: "${{ needs.report.outputs.commit_status_message }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: failure
target_url: ${{ needs.generate-test-cycle.outputs.status_check_url }}
+69 -5
View File
@@ -24,6 +24,28 @@ on:
type: string
required: false
description: "Server image tag (e.g., master or short SHA)"
server_edition:
type: string
required: false
description: "Server edition: enterprise (default), fips, or team"
server_image_repo:
type: string
required: false
default: mattermostdevelopment
description: "Docker registry: mattermostdevelopment (default) or mattermost"
skip_smoke:
type: boolean
required: false
default: false
description: "Skip smoke tests and run full tests directly"
server_image_aliases:
type: string
required: false
description: "Comma-separated alias tags for context name (e.g., 'release-11.4, release-11')"
ref_branch:
type: string
required: false
description: "Source branch name for webhook messages (e.g., 'master' or 'release-11.4')"
secrets:
MM_LICENSE:
required: false
@@ -47,6 +69,7 @@ jobs:
branch: "${{ steps.build-vars.outputs.branch }}"
build_id: "${{ steps.build-vars.outputs.build_id }}"
server_image_tag: "${{ steps.build-vars.outputs.server_image_tag }}"
server_image: "${{ steps.build-vars.outputs.server_image }}"
steps:
- name: ci/generate-build-variables
id: build-vars
@@ -63,19 +86,52 @@ jobs:
else
SERVER_IMAGE_TAG="${COMMIT_SHA::7}"
fi
# Validate server_image_tag format (alphanumeric, dots, hyphens, underscores)
if ! [[ "$SERVER_IMAGE_TAG" =~ ^[a-zA-Z0-9._-]+$ ]]; then
echo "::error::Invalid server_image_tag format: ${SERVER_IMAGE_TAG}"
exit 1
fi
echo "server_image_tag=${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
# Generate branch name
REF_BRANCH="${{ inputs.ref_branch }}"
if [ -n "$PR_NUMBER" ]; then
echo "branch=server-pr-${PR_NUMBER}" >> $GITHUB_OUTPUT
elif [ -n "$REF_BRANCH" ]; then
echo "branch=server-${REF_BRANCH}-${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
else
echo "branch=server-commit-${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
fi
# Determine server image name
EDITION="${{ inputs.server_edition }}"
REPO="${{ inputs.server_image_repo }}"
REPO="${REPO:-mattermostdevelopment}"
case "$EDITION" in
fips) IMAGE_NAME="mattermost-enterprise-fips-edition" ;;
team) IMAGE_NAME="mattermost-team-edition" ;;
*) IMAGE_NAME="mattermost-enterprise-edition" ;;
esac
SERVER_IMAGE="${REPO}/${IMAGE_NAME}:${SERVER_IMAGE_TAG}"
echo "server_image=${SERVER_IMAGE}" >> $GITHUB_OUTPUT
# Validate server_image_aliases format if provided
ALIASES="${{ inputs.server_image_aliases }}"
if [ -n "$ALIASES" ] && ! [[ "$ALIASES" =~ ^[a-zA-Z0-9._,\ -]+$ ]]; then
echo "::error::Invalid server_image_aliases format: ${ALIASES}"
exit 1
fi
# Generate build ID
echo "build_id=${RUN_ID}_${RUN_ATTEMPT}-${SERVER_IMAGE_TAG}-cypress-onprem-ent" >> $GITHUB_OUTPUT
if [ -n "$EDITION" ] && [ "$EDITION" != "enterprise" ]; then
echo "build_id=${RUN_ID}_${RUN_ATTEMPT}-${SERVER_IMAGE_TAG}-cypress-onprem-${EDITION}" >> $GITHUB_OUTPUT
else
echo "build_id=${RUN_ID}_${RUN_ATTEMPT}-${SERVER_IMAGE_TAG}-cypress-onprem-ent" >> $GITHUB_OUTPUT
fi
cypress-smoke:
if: ${{ !inputs.skip_smoke }}
needs:
- generate-build-variables
uses: ./.github/workflows/e2e-tests-cypress-template.yml
@@ -89,8 +145,11 @@ jobs:
branch: ${{ needs.generate-build-variables.outputs.branch }}
build_id: ${{ needs.generate-build-variables.outputs.build_id }}
server_image_tag: ${{ needs.generate-build-variables.outputs.server_image_tag }}
server_edition: ${{ inputs.server_edition }}
server_image_repo: ${{ inputs.server_image_repo }}
server_image_aliases: ${{ inputs.server_image_aliases }}
server: ${{ inputs.server }}
context_name: "E2E Tests / cypress-smoke"
context_name: "e2e-test/cypress-smoke/${{ inputs.server_edition || 'enterprise' }}"
secrets:
MM_LICENSE: ${{ secrets.MM_LICENSE }}
AUTOMATION_DASHBOARD_URL: ${{ secrets.AUTOMATION_DASHBOARD_URL }}
@@ -99,15 +158,16 @@ jobs:
CWS_URL: ${{ secrets.CWS_URL }}
CWS_EXTRA_HTTP_HEADERS: ${{ secrets.CWS_EXTRA_HTTP_HEADERS }}
# Full Tests (runs if smoke passed or skipped)
cypress-full:
needs:
- cypress-smoke
- generate-build-variables
if: needs.cypress-smoke.outputs.failed == '0'
if: always() && (needs.cypress-smoke.result == 'skipped' || needs.cypress-smoke.outputs.failed == '0')
uses: ./.github/workflows/e2e-tests-cypress-template.yml
with:
test_type: full
test_filter: '--stage="@prod" --excludeGroup="@smoke,@te_only,@cloud_only,@high_availability" --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap" --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"'
test_filter: '--stage="@prod" --excludeGroup="@te_only,@cloud_only,@high_availability" --sortFirst="@compliance_export,@elasticsearch,@ldap_group,@ldap" --sortLast="@saml,@keycloak,@plugin,@plugins_uninstall,@mfa,@license_removal"'
workers: 20
timeout_minutes: 60
enabled_docker_services: "postgres inbucket minio openldap elasticsearch keycloak"
@@ -115,11 +175,15 @@ jobs:
branch: ${{ needs.generate-build-variables.outputs.branch }}
build_id: ${{ needs.generate-build-variables.outputs.build_id }}
server_image_tag: ${{ needs.generate-build-variables.outputs.server_image_tag }}
server_edition: ${{ inputs.server_edition }}
server_image_repo: ${{ inputs.server_image_repo }}
server_image_aliases: ${{ inputs.server_image_aliases }}
server: ${{ inputs.server }}
enable_reporting: ${{ inputs.enable_reporting }}
report_type: ${{ inputs.report_type }}
ref_branch: ${{ inputs.ref_branch }}
pr_number: ${{ inputs.pr_number }}
context_name: "E2E Tests / cypress-full"
context_name: "e2e-test/cypress-full/${{ inputs.server_edition || 'enterprise' }}"
secrets:
MM_LICENSE: ${{ secrets.MM_LICENSE }}
AUTOMATION_DASHBOARD_URL: ${{ secrets.AUTOMATION_DASHBOARD_URL }}
+134
View File
@@ -0,0 +1,134 @@
---
name: E2E Tests (master/release - merge)
on:
workflow_dispatch:
inputs:
branch:
type: string
required: true
description: "Branch name (e.g., 'master' or 'release-11.4')"
commit_sha:
type: string
required: true
description: "Commit SHA to test"
server_image_tag:
type: string
required: true
description: "Docker image tag (e.g., 'abc1234_def5678' or 'master')"
jobs:
generate-build-variables:
runs-on: ubuntu-24.04
outputs:
report_type: "${{ steps.vars.outputs.report_type }}"
ref_branch: "${{ steps.vars.outputs.ref_branch }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ inputs.branch }}
fetch-depth: 50
- name: ci/generate-variables
id: vars
env:
BRANCH: ${{ inputs.branch }}
COMMIT_SHA: ${{ inputs.commit_sha }}
run: |
# Strip refs/heads/ prefix if present
BRANCH="${BRANCH#refs/heads/}"
# Validate branch is master or release-X.Y
if [[ "$BRANCH" == "master" ]]; then
echo "report_type=MASTER" >> $GITHUB_OUTPUT
elif [[ "$BRANCH" =~ ^release-[0-9]+\.[0-9]+$ ]]; then
echo "report_type=RELEASE" >> $GITHUB_OUTPUT
else
echo "::error::Branch ${BRANCH} must be 'master' or 'release-X.Y' format."
exit 1
fi
echo "ref_branch=${BRANCH}" >> $GITHUB_OUTPUT
# Validate commit exists on the branch
if ! git merge-base --is-ancestor "$COMMIT_SHA" HEAD; then
echo "::error::Commit ${COMMIT_SHA} is not on branch ${BRANCH}."
exit 1
fi
# Enterprise Edition
e2e-cypress:
needs: generate-build-variables
uses: ./.github/workflows/e2e-tests-cypress.yml
with:
commit_sha: ${{ inputs.commit_sha }}
server_image_tag: ${{ inputs.server_image_tag }}
skip_smoke: true
server: onprem
enable_reporting: true
report_type: ${{ needs.generate-build-variables.outputs.report_type }}
ref_branch: ${{ needs.generate-build-variables.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}"
PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
CWS_URL: "${{ secrets.MM_E2E_CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS }}"
e2e-playwright:
needs: generate-build-variables
uses: ./.github/workflows/e2e-tests-playwright.yml
with:
commit_sha: ${{ inputs.commit_sha }}
server_image_tag: ${{ inputs.server_image_tag }}
skip_smoke: true
server: onprem
enable_reporting: true
report_type: ${{ needs.generate-build-variables.outputs.report_type }}
ref_branch: ${{ needs.generate-build-variables.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
# Enterprise FIPS Edition
e2e-cypress-fips:
needs: generate-build-variables
uses: ./.github/workflows/e2e-tests-cypress.yml
with:
commit_sha: ${{ inputs.commit_sha }}
server_image_tag: ${{ inputs.server_image_tag }}
server_edition: fips
skip_smoke: true
server: onprem
enable_reporting: true
report_type: ${{ needs.generate-build-variables.outputs.report_type }}
ref_branch: ${{ needs.generate-build-variables.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}"
PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
CWS_URL: "${{ secrets.MM_E2E_CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS }}"
e2e-playwright-fips:
needs: generate-build-variables
uses: ./.github/workflows/e2e-tests-playwright.yml
with:
commit_sha: ${{ inputs.commit_sha }}
server_image_tag: ${{ inputs.server_image_tag }}
server_edition: fips
skip_smoke: true
server: onprem
enable_reporting: true
report_type: ${{ needs.generate-build-variables.outputs.report_type }}
ref_branch: ${{ needs.generate-build-variables.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
+137
View File
@@ -0,0 +1,137 @@
---
name: E2E Tests (release cut)
on:
workflow_dispatch:
inputs:
branch:
type: string
required: true
description: "Release branch (e.g., 'release-11.4')"
commit_sha:
type: string
required: true
description: "Commit SHA to test"
server_image_tag:
type: string
required: true
description: "Docker image tag (e.g., '11.4.0', '11.4.0-rc3', or 'release-11.4')"
server_image_aliases:
type: string
required: false
description: "Comma-separated alias tags (e.g., 'release-11.4, release-11')"
jobs:
validate:
runs-on: ubuntu-24.04
outputs:
ref_branch: "${{ steps.check.outputs.ref_branch }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ inputs.branch }}
fetch-depth: 50
- name: ci/validate-inputs
id: check
env:
BRANCH: ${{ inputs.branch }}
COMMIT_SHA: ${{ inputs.commit_sha }}
run: |
# Strip refs/heads/ prefix if present
BRANCH="${BRANCH#refs/heads/}"
if ! [[ "$BRANCH" =~ ^release-[0-9]+\.[0-9]+$ ]]; then
echo "::error::Branch ${BRANCH} must be 'release-X.Y' format."
exit 1
elif ! git merge-base --is-ancestor "$COMMIT_SHA" HEAD; then
echo "::error::Commit ${COMMIT_SHA} is not on branch ${BRANCH}."
exit 1
fi
echo "ref_branch=${BRANCH}" >> $GITHUB_OUTPUT
# Enterprise Edition
e2e-cypress:
needs: validate
uses: ./.github/workflows/e2e-tests-cypress.yml
with:
commit_sha: ${{ inputs.commit_sha }}
server_image_tag: ${{ inputs.server_image_tag }}
server_image_repo: mattermost
server_image_aliases: ${{ inputs.server_image_aliases }}
skip_smoke: true
server: onprem
enable_reporting: true
report_type: RELEASE_CUT
ref_branch: ${{ needs.validate.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}"
PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
CWS_URL: "${{ secrets.MM_E2E_CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS }}"
e2e-playwright:
needs: validate
uses: ./.github/workflows/e2e-tests-playwright.yml
with:
commit_sha: ${{ inputs.commit_sha }}
server_image_tag: ${{ inputs.server_image_tag }}
server_image_repo: mattermost
server_image_aliases: ${{ inputs.server_image_aliases }}
skip_smoke: true
server: onprem
enable_reporting: true
report_type: RELEASE_CUT
ref_branch: ${{ needs.validate.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
# Enterprise FIPS Edition
e2e-cypress-fips:
needs: validate
uses: ./.github/workflows/e2e-tests-cypress.yml
with:
commit_sha: ${{ inputs.commit_sha }}
server_image_tag: ${{ inputs.server_image_tag }}
server_edition: fips
server_image_repo: mattermost
server_image_aliases: ${{ inputs.server_image_aliases }}
skip_smoke: true
server: onprem
enable_reporting: true
report_type: RELEASE_CUT
ref_branch: ${{ needs.validate.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AUTOMATION_DASHBOARD_URL: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_URL }}"
AUTOMATION_DASHBOARD_TOKEN: "${{ secrets.MM_E2E_AUTOMATION_DASHBOARD_TOKEN }}"
PUSH_NOTIFICATION_SERVER: "${{ secrets.MM_E2E_PUSH_NOTIFICATION_SERVER }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
CWS_URL: "${{ secrets.MM_E2E_CWS_URL }}"
CWS_EXTRA_HTTP_HEADERS: "${{ secrets.MM_E2E_CWS_EXTRA_HTTP_HEADERS }}"
e2e-playwright-fips:
needs: validate
uses: ./.github/workflows/e2e-tests-playwright.yml
with:
commit_sha: ${{ inputs.commit_sha }}
server_image_tag: ${{ inputs.server_image_tag }}
server_edition: fips
server_image_repo: mattermost
server_image_aliases: ${{ inputs.server_image_aliases }}
skip_smoke: true
server: onprem
enable_reporting: true
report_type: RELEASE_CUT
ref_branch: ${{ needs.validate.outputs.ref_branch }}
secrets:
MM_LICENSE: "${{ secrets.MM_E2E_TEST_LICENSE_ONPREM_ENT }}"
AWS_ACCESS_KEY_ID: "${{ secrets.CYPRESS_AWS_ACCESS_KEY_ID }}"
AWS_SECRET_ACCESS_KEY: "${{ secrets.CYPRESS_AWS_SECRET_ACCESS_KEY }}"
REPORT_WEBHOOK_URL: "${{ secrets.MM_E2E_REPORT_WEBHOOK_URL }}"
@@ -37,7 +37,7 @@ jobs:
COMMIT_SHA: ${{ steps.pr-info.outputs.head_sha }}
run: |
# Only full tests can be overridden (smoke tests must pass)
FULL_TEST_CONTEXTS=("E2E Tests / playwright-full" "E2E Tests / cypress-full")
FULL_TEST_CONTEXTS=("e2e-test/playwright-full/enterprise" "e2e-test/cypress-full/enterprise")
for CONTEXT_NAME in "${FULL_TEST_CONTEXTS[@]}"; do
echo "Checking: $CONTEXT_NAME"
@@ -41,6 +41,20 @@ on:
type: string
required: false
default: onprem
server_edition:
description: "Server edition: enterprise (default), fips, or team"
type: string
required: false
default: enterprise
server_image_repo:
description: "Docker registry: mattermostdevelopment (default) or mattermost"
type: string
required: false
default: mattermostdevelopment
server_image_aliases:
description: "Comma-separated alias tags for description (e.g., 'release-11.4, release-11')"
type: string
required: false
# Reporting options
enable_reporting:
@@ -50,6 +64,10 @@ on:
report_type:
type: string
required: false
ref_branch:
description: "Source branch name for webhook messages (e.g., 'master' or 'release-11.4')"
type: string
required: false
pr_number:
type: string
required: false
@@ -82,7 +100,7 @@ on:
required: true
env:
SERVER_IMAGE: "mattermostdevelopment/mattermost-enterprise-edition:${{ inputs.server_image_tag }}"
SERVER_IMAGE: "${{ inputs.server_image_repo }}/${{ inputs.server_edition == 'fips' && 'mattermost-enterprise-fips-edition' || inputs.server_edition == 'team' && 'mattermost-team-edition' || 'mattermost-enterprise-edition' }}:${{ inputs.server_image_tag }}"
jobs:
update-initial-status:
@@ -96,7 +114,7 @@ jobs:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "with image tag: ${{ inputs.server_image_tag }}"
description: "tests running, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: pending
run-tests:
@@ -115,12 +133,12 @@ jobs:
BUILD_ID: "${{ inputs.build_id }}"
steps:
- name: ci/checkout-repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: ".nvmrc"
cache: npm
@@ -139,7 +157,7 @@ jobs:
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: playwright-${{ inputs.test_type }}-results
name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results
path: |
e2e-tests/playwright/logs/
e2e-tests/playwright/results/
@@ -166,14 +184,11 @@ jobs:
color: ${{ steps.calculate.outputs.color }}
steps:
- name: ci/checkout-repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: ci/download-results
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: playwright-${{ inputs.test_type }}-results
name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results
path: e2e-tests/playwright/
- name: ci/calculate
id: calculate
@@ -204,12 +219,12 @@ jobs:
BUILD_ID: "${{ inputs.build_id }}-retest"
steps:
- name: ci/checkout-repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
- name: ci/setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: ".nvmrc"
cache: npm
@@ -231,7 +246,7 @@ jobs:
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
if: always()
with:
name: playwright-${{ inputs.test_type }}-retest-results
name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-retest-results
path: |
e2e-tests/playwright/logs/
e2e-tests/playwright/results/
@@ -254,12 +269,9 @@ jobs:
working-directory: e2e-tests
steps:
- name: ci/checkout-repo
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
ref: ${{ inputs.commit_sha }}
fetch-depth: 0
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- name: ci/setup-node
uses: actions/setup-node@395ad3262231945c25e8478fd5baf05154b1d79f # v6.1.0
uses: actions/setup-node@6044e13b5dc448c55e2357c09f80417699197238 # v6.2.0
with:
node-version-file: ".nvmrc"
cache: npm
@@ -269,7 +281,7 @@ jobs:
- name: ci/download-results
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: playwright-${{ inputs.test_type }}-results
name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-results
path: e2e-tests/playwright/
# Download retest results (only if retest ran)
@@ -277,7 +289,7 @@ jobs:
if: needs.run-failed-tests.result != 'skipped'
uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
with:
name: playwright-${{ inputs.test_type }}-retest-results
name: playwright-${{ inputs.test_type }}-${{ inputs.server_edition }}-retest-results
path: e2e-tests/playwright/retest-results/
# Calculate results (with optional merge of retest results)
@@ -325,18 +337,29 @@ jobs:
if: inputs.enable_reporting && env.REPORT_WEBHOOK_URL != ''
env:
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
PASS_RATE: ${{ steps.final-results.outputs.pass_rate }}
PASSING: ${{ steps.final-results.outputs.passing }}
TOTAL: ${{ steps.final-results.outputs.total }}
TOTAL_SPECS: ${{ steps.final-results.outputs.total_specs }}
COMMIT_STATUS_MESSAGE: ${{ steps.final-results.outputs.commit_status_message }}
COLOR: ${{ steps.final-results.outputs.color }}
REPORT_URL: ${{ steps.upload-to-s3.outputs.report_url }}
TEST_TYPE: ${{ inputs.test_type }}
REPORT_TYPE: ${{ inputs.report_type }}
COMMIT_SHA: ${{ inputs.commit_sha }}
REF_BRANCH: ${{ inputs.ref_branch }}
PR_NUMBER: ${{ inputs.pr_number }}
run: |
# Capitalize test type
TEST_TYPE_CAP=$(echo "$TEST_TYPE" | sed 's/.*/\u&/')
# Build source line based on report type
COMMIT_SHORT="${COMMIT_SHA::7}"
COMMIT_URL="https://github.com/${{ github.repository }}/commit/${COMMIT_SHA}"
if [ "$REPORT_TYPE" = "RELEASE_CUT" ]; then
SOURCE_LINE=":github_round: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`"
elif [ "$REPORT_TYPE" = "MASTER" ] || [ "$REPORT_TYPE" = "RELEASE" ]; then
SOURCE_LINE=":git_merge: [${COMMIT_SHORT}](${COMMIT_URL}) on \`${REF_BRANCH}\`"
else
SOURCE_LINE=":open-pull-request: [mattermost-pr-${PR_NUMBER}](https://github.com/${{ github.repository }}/pull/${PR_NUMBER})"
fi
# Build payload with attachments
PAYLOAD=$(cat <<EOF
{
@@ -344,7 +367,7 @@ jobs:
"icon_url": "https://mattermost.com/wp-content/uploads/2022/02/icon_WS.png",
"attachments": [{
"color": "${COLOR}",
"text": "**Results - Playwright ${TEST_TYPE_CAP} Tests**\n\n:open-pull-request: [mattermost-pr-${PR_NUMBER}](https://github.com/${{ github.repository }}/pull/${PR_NUMBER})\n:docker: \`${{ env.SERVER_IMAGE }}\`\n${PASS_RATE}% (${PASSING}/${TOTAL}) in ${TOTAL_SPECS} spec files | [full report](${REPORT_URL})"
"text": "**Results - Playwright ${TEST_TYPE_CAP} Tests**\n\n${SOURCE_LINE}\n:docker: \`${{ env.SERVER_IMAGE }}\`\n${COMMIT_STATUS_MESSAGE} | [full report](${REPORT_URL})"
}]
}
EOF
@@ -419,7 +442,7 @@ jobs:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "${{ needs.report.outputs.commit_status_message }} with image tag: ${{ inputs.server_image_tag }}"
description: "${{ needs.report.outputs.commit_status_message }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: success
target_url: ${{ needs.report.outputs.report_url }}
@@ -437,6 +460,6 @@ jobs:
repository_full_name: ${{ github.repository }}
commit_sha: ${{ inputs.commit_sha }}
context: ${{ inputs.context_name }}
description: "${{ needs.report.outputs.commit_status_message }} with image tag: ${{ inputs.server_image_tag }}"
description: "${{ needs.report.outputs.commit_status_message }}, image_tag:${{ inputs.server_image_tag }}${{ inputs.server_image_aliases && format(' ({0})', inputs.server_image_aliases) || '' }}"
status: failure
target_url: ${{ needs.report.outputs.report_url }}
+69 -8
View File
@@ -24,6 +24,28 @@ on:
type: string
required: false
description: "Server image tag (e.g., master or short SHA)"
server_edition:
type: string
required: false
description: "Server edition: enterprise (default), fips, or team"
server_image_repo:
type: string
required: false
default: mattermostdevelopment
description: "Docker registry: mattermostdevelopment (default) or mattermost"
skip_smoke:
type: boolean
required: false
default: false
description: "Skip smoke tests and run full tests directly"
server_image_aliases:
type: string
required: false
description: "Comma-separated alias tags for context name (e.g., 'release-11.4, release-11')"
ref_branch:
type: string
required: false
description: "Source branch name for webhook messages (e.g., 'master' or 'release-11.4')"
secrets:
MM_LICENSE:
required: false
@@ -41,6 +63,7 @@ jobs:
branch: "${{ steps.build-vars.outputs.branch }}"
build_id: "${{ steps.build-vars.outputs.build_id }}"
server_image_tag: "${{ steps.build-vars.outputs.server_image_tag }}"
server_image: "${{ steps.build-vars.outputs.server_image }}"
steps:
- name: ci/generate-build-variables
id: build-vars
@@ -57,19 +80,52 @@ jobs:
else
SERVER_IMAGE_TAG="${COMMIT_SHA::7}"
fi
# Validate server_image_tag format (alphanumeric, dots, hyphens, underscores)
if ! [[ "$SERVER_IMAGE_TAG" =~ ^[a-zA-Z0-9._-]+$ ]]; then
echo "::error::Invalid server_image_tag format: ${SERVER_IMAGE_TAG}"
exit 1
fi
echo "server_image_tag=${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
# Generate branch name
REF_BRANCH="${{ inputs.ref_branch }}"
if [ -n "$PR_NUMBER" ]; then
echo "branch=server-pr-${PR_NUMBER}" >> $GITHUB_OUTPUT
elif [ -n "$REF_BRANCH" ]; then
echo "branch=server-${REF_BRANCH}-${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
else
echo "branch=server-commit-${SERVER_IMAGE_TAG}" >> $GITHUB_OUTPUT
fi
# Determine server image name
EDITION="${{ inputs.server_edition }}"
REPO="${{ inputs.server_image_repo }}"
REPO="${REPO:-mattermostdevelopment}"
case "$EDITION" in
fips) IMAGE_NAME="mattermost-enterprise-fips-edition" ;;
team) IMAGE_NAME="mattermost-team-edition" ;;
*) IMAGE_NAME="mattermost-enterprise-edition" ;;
esac
SERVER_IMAGE="${REPO}/${IMAGE_NAME}:${SERVER_IMAGE_TAG}"
echo "server_image=${SERVER_IMAGE}" >> $GITHUB_OUTPUT
# Validate server_image_aliases format if provided
ALIASES="${{ inputs.server_image_aliases }}"
if [ -n "$ALIASES" ] && ! [[ "$ALIASES" =~ ^[a-zA-Z0-9._,\ -]+$ ]]; then
echo "::error::Invalid server_image_aliases format: ${ALIASES}"
exit 1
fi
# Generate build ID
echo "build_id=${RUN_ID}_${RUN_ATTEMPT}-${SERVER_IMAGE_TAG}-playwright-onprem-ent" >> $GITHUB_OUTPUT
if [ -n "$EDITION" ] && [ "$EDITION" != "enterprise" ]; then
echo "build_id=${RUN_ID}_${RUN_ATTEMPT}-${SERVER_IMAGE_TAG}-playwright-onprem-${EDITION}" >> $GITHUB_OUTPUT
else
echo "build_id=${RUN_ID}_${RUN_ATTEMPT}-${SERVER_IMAGE_TAG}-playwright-onprem-ent" >> $GITHUB_OUTPUT
fi
playwright-smoke:
if: ${{ !inputs.skip_smoke }}
needs:
- generate-build-variables
uses: ./.github/workflows/e2e-tests-playwright-template.yml
@@ -82,37 +138,42 @@ jobs:
branch: ${{ needs.generate-build-variables.outputs.branch }}
build_id: ${{ needs.generate-build-variables.outputs.build_id }}
server_image_tag: ${{ needs.generate-build-variables.outputs.server_image_tag }}
server_edition: ${{ inputs.server_edition }}
server_image_repo: ${{ inputs.server_image_repo }}
server_image_aliases: ${{ inputs.server_image_aliases }}
server: ${{ inputs.server }}
context_name: "E2E Tests / playwright-smoke"
context_name: "e2e-test/playwright-smoke/${{ inputs.server_edition || 'enterprise' }}"
pr_number: ${{ inputs.pr_number }}
secrets:
MM_LICENSE: ${{ secrets.MM_LICENSE }}
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
# ════════════════════════════════════════════════════════════════════════════
# FULL TESTS (only if smoke passes and pr_number is provided)
# ════════════════════════════════════════════════════════════════════════════
# Full Tests (runs if smoke passed or skipped)
playwright-full:
needs:
- playwright-smoke
- generate-build-variables
if: needs.playwright-smoke.outputs.failed == '0' && inputs.pr_number != ''
if: always() && (needs.playwright-smoke.result == 'skipped' || needs.playwright-smoke.outputs.failed == '0')
uses: ./.github/workflows/e2e-tests-playwright-template.yml
with:
test_type: full
test_filter: '--grep-invert "@smoke|@visual"'
test_filter: '--grep-invert "@visual"'
timeout_minutes: 120
enabled_docker_services: "postgres inbucket minio openldap elasticsearch keycloak"
commit_sha: ${{ inputs.commit_sha }}
branch: ${{ needs.generate-build-variables.outputs.branch }}
build_id: ${{ needs.generate-build-variables.outputs.build_id }}
server_image_tag: ${{ needs.generate-build-variables.outputs.server_image_tag }}
server_edition: ${{ inputs.server_edition }}
server_image_repo: ${{ inputs.server_image_repo }}
server_image_aliases: ${{ inputs.server_image_aliases }}
server: ${{ inputs.server }}
enable_reporting: ${{ inputs.enable_reporting }}
report_type: ${{ inputs.report_type }}
ref_branch: ${{ inputs.ref_branch }}
pr_number: ${{ inputs.pr_number }}
context_name: "E2E Tests / playwright-full"
context_name: "e2e-test/playwright-full/${{ inputs.server_edition || 'enterprise' }}"
secrets:
MM_LICENSE: ${{ secrets.MM_LICENSE }}
REPORT_WEBHOOK_URL: ${{ secrets.REPORT_WEBHOOK_URL }}
@@ -34,7 +34,7 @@ jobs:
COMMIT_SHA: ${{ github.event.pull_request.head.sha }}
run: |
# Only full tests can be overridden (smoke tests must pass)
FULL_TEST_CONTEXTS=("E2E Tests / playwright-full" "E2E Tests / cypress-full")
FULL_TEST_CONTEXTS=("e2e-test/playwright-full/enterprise" "e2e-test/cypress-full/enterprise")
OVERRIDDEN=""
WEBHOOK_DATA="[]"