MM-66555: Add GH action to save mmctl E2E test report to Zephyr (#34429)

* add GH action to save mmctl e2e test report to zephyr * test on pr * bundle dependencies and set conditonal run on local and GH * ensure test keys are saved * improve github summary * add test, organize types * update dependencies * only run on master and release branch
2026-08-27 05:37:15 -05:00 · 2025-11-14 09:22:00 +08:00
parent bdcdff6a16
commit 5de5102b99
19 changed files with 22991 additions and 0 deletions
@@ -0,0 +1,18 @@
+# Example environment file for local testing
+# Copy this to .env and fill in your values
+# Note: GitHub Actions inputs use INPUT_ prefix with hyphens kept in the name
+
+# Required inputs
+INPUT_REPORT-PATH=./example-report/report.xml
+INPUT_ZEPHYR-API-KEY=your-api-key-here
+INPUT_BUILD-IMAGE=mattermostdevelopment/mattermost-enterprise-edition:1234567
+
+# GitHub environment variables (used to generate build-number internally)
+GITHUB_HEAD_REF=feature-branch
+GITHUB_REF_NAME=master
+GITHUB_RUN_ID=12345678
+GITHUB_REPOSITORY=mattermost/mattermost
+
+# Optional inputs with defaults
+INPUT_ZEPHYR-FOLDER-ID=27504432
+INPUT_JIRA-PROJECT-KEY=MM
@@ -0,0 +1,4 @@
+.env
+package-lock.json
+
+.example-report
@@ -0,0 +1,77 @@
+# Save JUnit Test Report to TMS Action
+
+GitHub Action to save JUnit test reports to Zephyr Scale Test Management System.
+
+## Usage
+
+```yaml
+- name: Save JUnit test report to Zephyr
+  uses: ./.github/actions/save-junit-report-tms
+  with:
+    report-path: ./test-reports/report.xml
+    zephyr-api-key: ${{ secrets.ZEPHYR_API_KEY }}
+    build-image: ${{ env.BUILD_IMAGE }}
+    zephyr-folder-id: '27504432'  # Optional, defaults to 27504432
+    jira-project-key: 'MM'  # Optional, defaults to MM
+```
+
+## Inputs
+
+| Input | Description | Required | Default |
+|-------|-------------|----------|---------|
+| `report-path` | Path to the XML test report file (from artifact) | Yes | - |
+| `zephyr-api-key` | Zephyr Scale API key | Yes | - |
+| `build-image` | Docker build image used for testing | Yes | - |
+| `zephyr-folder-id` | Zephyr Scale folder ID | No | `27504432` |
+| `jira-project-key` | Jira project key | No | `MM` |
+
+## Outputs
+
+| Output | Description |
+|--------|-------------|
+| `test-cycle` | The created test cycle key in Zephyr Scale |
+| `test-keys-execution-count` | Total number of test executions (including duplicates) |
+| `test-keys-unique-count` | Number of unique test keys successfully saved to Zephyr |
+| `junit-total-tests` | Total number of tests in the JUnit XML report |
+| `junit-total-passed` | Number of passed tests in the JUnit XML report |
+| `junit-total-failed` | Number of failed tests in the JUnit XML report |
+| `junit-pass-rate` | Pass rate percentage from the JUnit XML report |
+| `junit-duration-seconds` | Total test duration in seconds from the JUnit XML report |
+
+## Local Development
+
+1. Copy `.env.example` to `.env` and fill in your values
+2. Run `npm install` to install dependencies
+3. Run `npm run pretter` to format code
+4. Run `npm test` to run unit tests
+5. Run `npm run local-action` to test locally
+6. Run `npm run build` to build for production
+
+### Submitting Code Changes
+
+**IMPORTANT**: When submitting code changes, you must run the following checks locally as there are no CI jobs for this action:
+
+1. Run `npm run prettier` to format your code
+2. Run `npm test` to ensure all tests pass
+3. Run `npm run build` to compile your changes
+4. Include the updated `dist/` folder in your commit
+
+GitHub Actions runs the compiled code from the `dist/` folder, not the source TypeScript files. If you don't include the built files, your changes won't be reflected in the action.
+
+## Report Format
+
+The action expects a JUnit XML format report with test case names containing Zephyr test keys in the format `{PROJECT_KEY}-T{NUMBER}` (e.g., `MM-T1234`, `FOO-T5678`).
+
+The test key pattern is automatically determined by the `jira-project-key` input (defaults to `MM`).
+
+Example:
+```xml
+<testsuites tests="10" failures="2" errors="0" time="45.2">
+  <testsuite name="mmctl tests" tests="10" failures="2" time="45.2" timestamp="2024-01-01T00:00:00Z">
+    <testcase name="MM-T1234 - Test user creation" time="2.5"/>
+    <testcase name="MM-T1235 - Test user login" time="3.2">
+      <failure message="Login failed"/>
+    </testcase>
+  </testsuite>
+</testsuites>
+```
@@ -0,0 +1,46 @@
+name: Save JUnit Test Report to TMS
+description: Save JUnit test report to Zephyr Scale Test Management System
+author: Mattermost
+
+# Define your inputs here.
+inputs:
+  report-path:
+    description: Path to the XML test report file (from artifact)
+    required: true
+  zephyr-api-key:
+    description: Zephyr Scale API key
+    required: true
+  build-image:
+    description: Docker build image used for testing
+    required: true
+  zephyr-folder-id:
+    description: Zephyr Scale folder ID
+    required: false
+    default: '27504432'
+  jira-project-key:
+    description: Jira project key
+    required: false
+    default: 'MM'
+
+# Define your outputs here.
+outputs:
+  test-cycle:
+    description: The created test cycle key in Zephyr Scale
+  test-keys-execution-count:
+    description: Total number of test executions (including duplicates)
+  test-keys-unique-count:
+    description: Number of unique test keys successfully saved to Zephyr
+  junit-total-tests:
+    description: Total number of tests in the JUnit XML report
+  junit-total-passed:
+    description: Number of passed tests in the JUnit XML report
+  junit-total-failed:
+    description: Number of failed tests in the JUnit XML report
+  junit-pass-rate:
+    description: Pass rate percentage from the JUnit XML report
+  junit-duration-seconds:
+    description: Total test duration in seconds from the JUnit XML report
+
+runs:
+  using: node24
+  main: dist/index.js
@@ -0,0 +1,14 @@
+module.exports = {
+  preset: 'ts-jest',
+  testEnvironment: 'node',
+  testMatch: ['**/__tests__/**/*.test.ts'],
+  collectCoverageFrom: [
+    'src/**/*.ts',
+    '!src/**/*.d.ts',
+    '!src/index.ts',
+  ],
+  moduleFileExtensions: ['ts', 'js', 'json'],
+  verbose: true,
+  // Suppress console output from code during tests
+  setupFilesAfterEnv: ['<rootDir>/jest.setup.js'],
+};
@@ -0,0 +1,19 @@
+// Mock @actions/core to suppress console output during tests
+jest.mock('@actions/core', () => ({
+  info: jest.fn(),
+  warning: jest.fn(),
+  error: jest.fn(),
+  debug: jest.fn(),
+  startGroup: jest.fn(),
+  endGroup: jest.fn(),
+  setOutput: jest.fn(),
+  setFailed: jest.fn(),
+  getInput: jest.fn(),
+  summary: {
+    addHeading: jest.fn().mockReturnThis(),
+    addTable: jest.fn().mockReturnThis(),
+    addLink: jest.fn().mockReturnThis(),
+    addRaw: jest.fn().mockReturnThis(),
+    write: jest.fn().mockResolvedValue(undefined),
+  },
+}));
@@ -0,0 +1,26 @@
+{
+  "name": "save-junit-report-tms",
+  "private": true,
+  "version": "0.1.0",
+  "main": "dist/index.js",
+  "scripts": {
+    "build": "tsup",
+    "prettier": "npx prettier --write \"src/**/*.ts\"",
+    "local-action": "local-action . src/main.ts .env",
+    "test": "jest --verbose",
+    "test:watch": "jest --watch --verbose",
+    "test:silent": "jest --silent"
+  },
+  "dependencies": {
+    "@actions/core": "1.11.1",
+    "fast-xml-parser": "5.3.1"
+  },
+  "devDependencies": {
+    "@github/local-action": "6.0.2",
+    "@types/jest": "30.0.0",
+    "jest": "30.2.0",
+    "ts-jest": "29.4.5",
+    "tsup": "8.5.0",
+    "typescript": "5.9.3"
+  }
+}
@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="5" failures="1" errors="0" skipped="1" time="10.5">
+  <testsuite name="Test Suite 1" tests="5" failures="1" errors="0" skipped="1" time="10.5" timestamp="2024-01-15T10:00:00Z">
+    <testcase name="MM-T1001 User can login" classname="auth" time="2.5"/>
+    <testcase name="MM-T1002 User can logout" classname="auth" time="1.5"/>
+    <testcase name="MM-T1003 User can reset password" classname="auth" time="3.0">
+      <failure message="Password reset failed">Expected password to be reset but it wasn't</failure>
+    </testcase>
+    <testcase name="MM-T1004 User can view profile" classname="profile" time="2.0">
+      <skipped message="Test skipped"/>
+    </testcase>
+    <testcase name="MM-T1005 User can update profile" classname="profile" time="1.5"/>
+  </testsuite>
+</testsuites>
@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="6" failures="2" errors="0" skipped="0" time="15.0">
+  <testsuite name="Duplicate Key Tests" tests="6" failures="2" errors="0" skipped="0" time="15.0" timestamp="2024-01-15T11:00:00Z">
+    <testcase name="MM-T4001 Test with retry - attempt 1" classname="retry" time="2.0">
+      <failure message="First attempt failed"/>
+    </testcase>
+    <testcase name="MM-T4001 Test with retry - attempt 2" classname="retry" time="2.5"/>
+    <testcase name="MM-T4002 Another test - run 1" classname="retry" time="3.0"/>
+    <testcase name="MM-T4002 Another test - run 2" classname="retry" time="3.5">
+      <failure message="Second run failed"/>
+    </testcase>
+    <testcase name="MM-T4003 Single test" classname="single" time="2.0"/>
+    <testcase name="Test without MM key" classname="nokey" time="2.0"/>
+  </testsuite>
+</testsuites>
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="10" failures="2" errors="1" skipped="1" time="25.75">
+  <testsuite name="Authentication Tests" tests="5" failures="1" errors="0" skipped="0" time="12.5" timestamp="2024-01-15T10:00:00Z">
+    <testcase name="MM-T2001 Admin can login" classname="auth" time="2.5"/>
+    <testcase name="MM-T2002 Guest can login" classname="auth" time="2.0"/>
+    <testcase name="MM-T2003 Invalid credentials rejected" classname="auth" time="3.5">
+      <failure message="Test assertion failed">Expected rejection but got acceptance</failure>
+    </testcase>
+    <testcase name="MM-T2004 Session expires correctly" classname="auth" time="2.5"/>
+    <testcase name="MM-T2005 Token refresh works" classname="auth" time="2.0"/>
+  </testsuite>
+  <testsuite name="Channel Tests" tests="5" failures="1" errors="1" skipped="1" time="13.25" timestamp="2024-01-15T10:05:00Z">
+    <testcase name="MM-T3001 Create channel" classname="channel" time="1.5"/>
+    <testcase name="MM-T3002 Delete channel" classname="channel" time="2.0">
+      <failure message="Channel not deleted">Expected channel to be deleted</failure>
+    </testcase>
+    <testcase name="MM-T3003 Archive channel" classname="channel" time="1.75">
+      <error message="Unexpected error">NullPointerException at line 45</error>
+    </testcase>
+    <testcase name="MM-T3004 Restore channel" classname="channel" time="3.0">
+      <skipped message="Feature not ready"/>
+    </testcase>
+    <testcase name="MM-T3005 Rename channel" classname="channel" time="5.0"/>
+  </testsuite>
+</testsuites>
@@ -0,0 +1,114 @@
+import { sortTestExecutions } from "../main";
+import type { TestExecution } from "../types";
+
+describe("sortTestExecutions", () => {
+    it("should sort by status first (Pass, Fail, Not Executed)", () => {
+        const executions: TestExecution[] = [
+            {
+                testCaseKey: "MM-T1003",
+                statusName: "Fail",
+                executionTime: 3.0,
+                comment: "Test 3",
+            },
+            {
+                testCaseKey: "MM-T1001",
+                statusName: "Pass",
+                executionTime: 1.0,
+                comment: "Test 1",
+            },
+            {
+                testCaseKey: "MM-T1004",
+                statusName: "Not Executed",
+                executionTime: 4.0,
+                comment: "Test 4",
+            },
+            {
+                testCaseKey: "MM-T1002",
+                statusName: "Pass",
+                executionTime: 2.0,
+                comment: "Test 2",
+            },
+        ];
+
+        const sorted = sortTestExecutions(executions);
+
+        // All Pass should come first
+        expect(sorted[0].statusName).toBe("Pass");
+        expect(sorted[1].statusName).toBe("Pass");
+        // Then Fail
+        expect(sorted[2].statusName).toBe("Fail");
+        // Then Not Executed
+        expect(sorted[3].statusName).toBe("Not Executed");
+    });
+
+    it("should sort by test key within same status", () => {
+        const executions: TestExecution[] = [
+            {
+                testCaseKey: "MM-T1003",
+                statusName: "Pass",
+                executionTime: 3.0,
+                comment: "Test 3",
+            },
+            {
+                testCaseKey: "MM-T1001",
+                statusName: "Pass",
+                executionTime: 1.0,
+                comment: "Test 1",
+            },
+            {
+                testCaseKey: "MM-T1002",
+                statusName: "Pass",
+                executionTime: 2.0,
+                comment: "Test 2",
+            },
+        ];
+
+        const sorted = sortTestExecutions(executions);
+
+        expect(sorted[0].testCaseKey).toBe("MM-T1001");
+        expect(sorted[1].testCaseKey).toBe("MM-T1002");
+        expect(sorted[2].testCaseKey).toBe("MM-T1003");
+    });
+
+    it("should not mutate the original array", () => {
+        const executions: TestExecution[] = [
+            {
+                testCaseKey: "MM-T1002",
+                statusName: "Fail",
+                executionTime: 2.0,
+                comment: "Test 2",
+            },
+            {
+                testCaseKey: "MM-T1001",
+                statusName: "Pass",
+                executionTime: 1.0,
+                comment: "Test 1",
+            },
+        ];
+
+        const sorted = sortTestExecutions(executions);
+
+        // Original array should remain unchanged
+        expect(executions[0].testCaseKey).toBe("MM-T1002");
+        expect(sorted[0].testCaseKey).toBe("MM-T1001");
+    });
+
+    it("should handle empty array", () => {
+        const sorted = sortTestExecutions([]);
+        expect(sorted).toEqual([]);
+    });
+
+    it("should handle single item", () => {
+        const executions: TestExecution[] = [
+            {
+                testCaseKey: "MM-T1001",
+                statusName: "Pass",
+                executionTime: 1.0,
+                comment: "Test 1",
+            },
+        ];
+
+        const sorted = sortTestExecutions(executions);
+        expect(sorted).toEqual(executions);
+    });
+});
@@ -0,0 +1,285 @@
+import * as path from "path";
+import { getTestData } from "../main";
+
+describe("getTestData", () => {
+    const config = {
+        projectKey: "MM",
+        zephyrFolderId: 27504432,
+        branch: "feature-branch",
+        buildImage:
+            "mattermostdevelopment/mattermost-enterprise-edition:1234567",
+        buildNumber: "feature-branch-12345678",
+        githubRunUrl:
+            "https://github.com/mattermost/mattermost/actions/runs/12345678",
+    };
+
+    describe("Basic JUnit report parsing", () => {
+        it("should parse a basic JUnit report correctly", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-basic.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            // Check test cycle metadata
+            expect(result.testCycle.projectKey).toBe("MM");
+            expect(result.testCycle.name).toBe(
+                "mmctl: E2E Tests with feature-branch, mattermostdevelopment/mattermost-enterprise-edition:1234567, feature-branch-12345678",
+            );
+            expect(result.testCycle.statusName).toBe("Done");
+            expect(result.testCycle.folderId).toBe(27504432);
+            expect(result.testCycle.description).toContain("Test Summary:");
+            expect(result.testCycle.description).toContain(
+                "github.com/mattermost/mattermost/actions/runs",
+            );
+
+            // Check JUnit stats
+            expect(result.junitStats.totalTests).toBe(5);
+            expect(result.junitStats.totalFailures).toBe(1);
+            expect(result.junitStats.totalErrors).toBe(0);
+            expect(result.junitStats.totalSkipped).toBe(1);
+            expect(result.junitStats.totalPassed).toBe(4);
+            expect(result.junitStats.totalTime).toBe(10.5);
+            expect(result.junitStats.passRate).toBe("80.0");
+
+            // Check test key stats
+            expect(result.testKeyStats.totalOccurrences).toBe(5);
+            expect(result.testKeyStats.uniqueCount).toBe(5);
+            expect(result.testKeyStats.passedCount).toBe(3);
+            expect(result.testKeyStats.failedCount).toBe(1);
+            expect(result.testKeyStats.skippedCount).toBe(1);
+            expect(result.testKeyStats.failedKeys).toEqual(["MM-T1003"]);
+            expect(result.testKeyStats.skippedKeys).toEqual(["MM-T1004"]);
+
+            // Check test executions
+            expect(result.testExecutions).toHaveLength(5);
+            expect(result.testExecutions[0].testCaseKey).toBe("MM-T1001");
+            expect(result.testExecutions[0].statusName).toBe("Pass");
+            expect(result.testExecutions[2].testCaseKey).toBe("MM-T1003");
+            expect(result.testExecutions[2].statusName).toBe("Fail");
+            expect(result.testExecutions[3].testCaseKey).toBe("MM-T1004");
+            expect(result.testExecutions[3].statusName).toBe("Not Executed");
+        });
+
+        it("should handle timestamps and set planned dates", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-basic.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            expect(result.testCycle.plannedStartDate).toBeDefined();
+            expect(result.testCycle.plannedEndDate).toBeDefined();
+
+            const startDate = new Date(result.testCycle.plannedStartDate!);
+            const endDate = new Date(result.testCycle.plannedEndDate!);
+
+            expect(startDate.getTime()).toBeLessThanOrEqual(endDate.getTime());
+        });
+    });
+
+    describe("Multiple test suites", () => {
+        it("should aggregate stats from multiple testsuites", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-multiple-suites.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            // Aggregated stats
+            expect(result.junitStats.totalTests).toBe(10);
+            expect(result.junitStats.totalFailures).toBe(2);
+            expect(result.junitStats.totalErrors).toBe(1);
+            expect(result.junitStats.totalSkipped).toBe(1);
+            expect(result.junitStats.totalTime).toBe(25.75);
+
+            // Test executions extracted
+            expect(result.testExecutions).toHaveLength(10);
+            expect(result.testKeyStats.uniqueCount).toBe(10);
+        });
+
+        it("should handle latest timestamp from multiple suites", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-multiple-suites.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            expect(result.testCycle.plannedStartDate).toBeDefined();
+            expect(result.testCycle.plannedEndDate).toBeDefined();
+
+            const startDate = new Date(result.testCycle.plannedStartDate!);
+            const endDate = new Date(result.testCycle.plannedEndDate!);
+
+            // End date should be after start date
+            expect(endDate.getTime()).toBeGreaterThan(startDate.getTime());
+        });
+    });
+
+    describe("Duplicate test keys", () => {
+        it("should count test key occurrences separately from unique keys", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-duplicate-keys.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            // Total test executions created (including duplicates)
+            expect(result.testExecutions).toHaveLength(5); // 5 tests with MM-T keys
+
+            // Test key statistics
+            expect(result.testKeyStats.totalOccurrences).toBe(5);
+            expect(result.testKeyStats.uniqueCount).toBe(3); // MM-T4001, MM-T4002, MM-T4003
+
+            // Status tracking for unique keys
+            // MM-T4001: has both Pass and Fail
+            // MM-T4002: has both Pass and Fail
+            // MM-T4003: has only Pass
+            expect(result.testKeyStats.passedCount).toBe(3); // MM-T4001, MM-T4002, MM-T4003
+            expect(result.testKeyStats.failedCount).toBe(2); // MM-T4001, MM-T4002
+            expect(result.testKeyStats.skippedCount).toBe(0);
+        });
+
+        it("should create separate test executions for each occurrence", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-duplicate-keys.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            // Find MM-T4001 executions
+            const t4001Executions = result.testExecutions.filter(
+                (e) => e.testCaseKey === "MM-T4001",
+            );
+            expect(t4001Executions).toHaveLength(2);
+
+            // One should be Fail, one should be Pass
+            const statuses = t4001Executions.map((e) => e.statusName).sort();
+            expect(statuses).toEqual(["Fail", "Pass"]);
+        });
+    });
+
+    describe("Test execution data", () => {
+        it("should return executions with correct status values", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-duplicate-keys.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            // Verify we have both Pass and Fail statuses
+            const statuses = result.testExecutions.map((e) => e.statusName);
+            expect(statuses).toContain("Pass");
+            expect(statuses).toContain("Fail");
+
+            // Verify each execution has required fields
+            result.testExecutions.forEach((execution) => {
+                expect(execution.testCaseKey).toBeTruthy();
+                expect(execution.statusName).toBeTruthy();
+                expect(execution.comment).toBeTruthy();
+                expect(typeof execution.executionTime).toBe("number");
+            });
+        });
+    });
+
+    describe("Test cycle description", () => {
+        it("should include test summary in description", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-basic.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            expect(result.testCycle.description).toContain("Test Summary:");
+            expect(result.testCycle.description).toContain("4 passed");
+            expect(result.testCycle.description).toContain("1 failed");
+            expect(result.testCycle.description).toContain("80.0% pass rate");
+            expect(result.testCycle.description).toContain("10.5s duration");
+        });
+
+        it("should include build details in description", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-basic.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            expect(result.testCycle.description).toContain(
+                "branch: feature-branch",
+            );
+            expect(result.testCycle.description).toContain(
+                "build image: mattermostdevelopment/mattermost-enterprise-edition:1234567",
+            );
+            expect(result.testCycle.description).toContain(
+                "build number: feature-branch-12345678",
+            );
+        });
+
+        it("should include GitHub run URL", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-basic.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            expect(result.testCycle.description).toContain(
+                "https://github.com/mattermost/mattermost/actions/runs/12345678",
+            );
+        });
+    });
+
+    describe("Edge cases", () => {
+        it("should handle reports with no test keys", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-duplicate-keys.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            // Report has 6 total tests, but only 5 have MM-T keys
+            expect(result.junitStats.totalTests).toBe(6);
+            expect(result.testExecutions).toHaveLength(5);
+        });
+
+        it("should calculate 0% pass rate when all tests fail", async () => {
+            // Using report-multiple-suites which has failures
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "report-multiple-suites.xml",
+            );
+            const result = await getTestData(reportPath, config);
+
+            // Pass rate should be less than 100%
+            expect(parseFloat(result.junitStats.passRate)).toBeLessThan(100);
+        });
+    });
+
+    describe("Error handling", () => {
+        it("should throw error for non-existent file", async () => {
+            const reportPath = path.join(
+                __dirname,
+                "fixtures",
+                "non-existent.xml",
+            );
+
+            await expect(getTestData(reportPath, config)).rejects.toThrow();
+        });
+
+        it("should throw error for invalid XML", async () => {
+            // This would require creating an invalid XML fixture
+            // Skipping for now, but you could add one
+        });
+    });
+});
@@ -0,0 +1,3 @@
+import { run } from "./main";
+
+run();
@@ -0,0 +1,563 @@
+import * as core from "@actions/core";
+import * as fs from "fs/promises";
+import { XMLParser } from "fast-xml-parser";
+import type {
+    TestExecution,
+    TestCycle,
+    TestData,
+    ZephyrApiClient,
+} from "./types";
+
+const zephyrCloudApiUrl = "https://api.zephyrscale.smartbear.com/v2";
+
+function newTestExecution(
+    testCaseKey: string,
+    statusName: string,
+    executionTime: number,
+    comment: string,
+): TestExecution {
+    return {
+        testCaseKey,
+        statusName, // Pass or Fail
+        executionTime,
+        comment,
+    };
+}
+
+export async function getTestData(
+    junitFile: string,
+    config: {
+        projectKey: string;
+        zephyrFolderId: number;
+        branch: string;
+        buildImage: string;
+        buildNumber: string;
+        githubRunUrl: string;
+    },
+): Promise<TestData> {
+    const testCycle: TestCycle = {
+        projectKey: config.projectKey,
+        name: `mmctl: E2E Tests with ${config.branch}, ${config.buildImage}, ${config.buildNumber}`,
+        description: "",
+        statusName: "Done",
+        folderId: config.zephyrFolderId,
+    };
+
+    // Create dynamic regex based on project key (e.g., MM-T1234, FOO-T5678)
+    const reTestKey = new RegExp(`${config.projectKey}-T\\d+`);
+    const testExecutions: TestExecution[] = [];
+
+    const data = await fs.readFile(junitFile, "utf8");
+
+    const parser = new XMLParser({
+        ignoreAttributes: false,
+        attributeNamePrefix: "@_",
+    });
+
+    const result = parser.parse(data);
+
+    // Parse test results and collect test data
+    let totalTests = 0;
+    let totalFailures = 0;
+    let totalErrors = 0;
+    let totalSkipped = 0;
+    let totalTime = 0;
+    let earliestTimestamp: Date | null = null;
+    let latestTimestamp: Date | null = null;
+
+    // Track test keys by status
+    const testKeysPassed = new Set<string>();
+    const testKeysFailed = new Set<string>();
+    const testKeysSkipped = new Set<string>();
+    let totalTestKeyOccurrences = 0;
+
+    if (result?.testsuites?.testsuite) {
+        const testsuites = Array.isArray(result.testsuites.testsuite)
+            ? result.testsuites.testsuite
+            : [result.testsuites.testsuite];
+
+        for (const testsuite of testsuites) {
+            const tests = parseInt(testsuite["@_tests"] || "0", 10);
+            const failures = parseInt(testsuite["@_failures"] || "0", 10);
+            const errors = parseInt(testsuite["@_errors"] || "0", 10);
+            const skipped = parseInt(testsuite["@_skipped"] || "0", 10);
+            const time = parseFloat(testsuite["@_time"] || "0");
+
+            totalTests += tests;
+            totalFailures += failures;
+            totalErrors += errors;
+            totalSkipped += skipped;
+            totalTime += time;
+
+            // Extract timestamp if available
+            const timestamp = testsuite["@_timestamp"];
+            if (timestamp) {
+                const date = new Date(timestamp);
+                if (!isNaN(date.getTime())) {
+                    if (!earliestTimestamp || date < earliestTimestamp) {
+                        earliestTimestamp = date;
+                    }
+                    if (!latestTimestamp || date > latestTimestamp) {
+                        latestTimestamp = date;
+                    }
+                }
+            }
+
+            if (testsuite?.testcase) {
+                const testcases = Array.isArray(testsuite.testcase)
+                    ? testsuite.testcase
+                    : [testsuite.testcase];
+
+                for (const testcase of testcases) {
+                    const testName = testcase["@_name"];
+                    const testTime = testcase["@_time"] || 0;
+                    const hasFailure = testcase.failure !== undefined;
+                    const hasSkipped = testcase.skipped !== undefined;
+
+                    if (testName) {
+                        const match = testName.match(reTestKey);
+
+                        if (match !== null) {
+                            const testKey = match[0];
+                            totalTestKeyOccurrences++;
+                            testCycle.description += `* ${testKey} - ${testTime}s\n`;
+
+                            let statusName: string;
+                            if (hasSkipped) {
+                                statusName = "Not Executed";
+                                testKeysSkipped.add(testKey);
+                            } else if (hasFailure) {
+                                statusName = "Fail";
+                                testKeysFailed.add(testKey);
+                            } else {
+                                statusName = "Pass";
+                                testKeysPassed.add(testKey);
+                            }
+
+                            testExecutions.push(
+                                newTestExecution(
+                                    testKey,
+                                    statusName,
+                                    parseFloat(testTime),
+                                    testName,
+                                ),
+                            );
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    // Log detailed summary
+    core.startGroup("JUnit report summary");
+    core.info(`  - Total tests: ${totalTests}`);
+    core.info(`  - Failures: ${totalFailures}`);
+    core.info(`  - Errors: ${totalErrors}`);
+    core.info(`  - Skipped: ${totalSkipped}`);
+    const timeInMinutes = (totalTime / 60).toFixed(1);
+    core.info(`  - Duration: ${totalTime.toFixed(1)}s (~${timeInMinutes}m)`);
+    core.endGroup();
+
+    core.startGroup("Extracted MM-T test cases");
+    const uniqueTestKeys = new Set([
+        ...testKeysPassed,
+        ...testKeysFailed,
+        ...testKeysSkipped,
+    ]);
+    core.info(`  - Total test key occurrences: ${totalTestKeyOccurrences}`);
+    core.info(`  - Unique test keys: ${uniqueTestKeys.size}`);
+    core.info(`  - Passed: ${testKeysPassed.size} test keys`);
+    if (testKeysFailed.size > 0) {
+        core.info(
+            `  - Failed: ${testKeysFailed.size} test keys (${Array.from(testKeysFailed).join(", ")})`,
+        );
+    } else {
+        core.info(`  - Failed: ${testKeysFailed.size} test keys`);
+    }
+    if (testKeysSkipped.size > 0) {
+        core.info(
+            `  - Skipped: ${testKeysSkipped.size} test keys (${Array.from(testKeysSkipped).join(", ")})`,
+        );
+    } else {
+        core.info(`  - Skipped: ${testKeysSkipped.size} test keys`);
+    }
+    core.endGroup();
+
+    // Build the description with summary and link
+    const passedTests = totalTests - totalFailures;
+    const passRate =
+        totalTests > 0 ? ((passedTests / totalTests) * 100).toFixed(1) : "0";
+
+    testCycle.description = `Test Summary: `;
+    testCycle.description += `${passedTests} passed | `;
+    testCycle.description += `${totalFailures} failed | `;
+    testCycle.description += `${passRate}% pass rate | `;
+    testCycle.description += `${totalTime.toFixed(1)}s duration | `;
+
+    testCycle.description += `branch: ${config.branch} | `;
+    testCycle.description += `build image: ${config.buildImage} | `;
+    testCycle.description += `build number: ${config.buildNumber} | `;
+
+    testCycle.description += `${config.githubRunUrl}`;
+
+    // Calculate and set planned start and end dates
+    if (earliestTimestamp) {
+        // Use the earliest timestamp from the report as the start date
+        testCycle.plannedStartDate = earliestTimestamp.toISOString();
+
+        // Calculate end date: if we have a latest timestamp, use it
+        // Otherwise, calculate from start + total duration
+        if (latestTimestamp && latestTimestamp > earliestTimestamp) {
+            testCycle.plannedEndDate = latestTimestamp.toISOString();
+        } else {
+            // Add total duration (in seconds) to start time
+            const endDate = new Date(
+                earliestTimestamp.getTime() + totalTime * 1000,
+            );
+            testCycle.plannedEndDate = endDate.toISOString();
+        }
+    }
+
+    return {
+        testCycle,
+        testExecutions,
+        junitStats: {
+            totalTests,
+            totalFailures,
+            totalErrors,
+            totalSkipped,
+            totalPassed: passedTests,
+            passRate,
+            totalTime,
+        },
+        testKeyStats: {
+            totalOccurrences: totalTestKeyOccurrences,
+            uniqueCount: uniqueTestKeys.size,
+            passedCount: testKeysPassed.size,
+            failedCount: testKeysFailed.size,
+            skippedCount: testKeysSkipped.size,
+            failedKeys: Array.from(testKeysFailed),
+            skippedKeys: Array.from(testKeysSkipped),
+        },
+    };
+}
+
+// Sort test executions by status (Pass, Fail, Not Executed), then by test key
+export function sortTestExecutions(
+    executions: TestExecution[],
+): TestExecution[] {
+    const statusOrder: Record<string, number> = {
+        Pass: 1,
+        Fail: 2,
+        "Not Executed": 3,
+    };
+
+    return [...executions].sort((a, b) => {
+        // First, sort by status
+        const statusA = statusOrder[a.statusName] || 999;
+        const statusB = statusOrder[b.statusName] || 999;
+        const statusComparison = statusA - statusB;
+        if (statusComparison !== 0) {
+            return statusComparison;
+        }
+
+        // Then, sort by test key
+        return a.testCaseKey.localeCompare(b.testCaseKey);
+    });
+}
+
+// Write GitHub Actions summary using core.summary API
+export async function writeGitHubSummary(
+    testCycle: TestCycle,
+    junitStats: TestData["junitStats"],
+    testKeyStats: TestData["testKeyStats"],
+    successCount: number,
+    failureCount: number,
+    uniqueSavedTestKeys: Set<string>,
+    uniqueFailedTestKeys: Set<string>,
+    projectKey: string,
+    testCycleKey: string,
+): Promise<void> {
+    const timeInMinutes = (junitStats.totalTime / 60).toFixed(1);
+    const zephyrUrl = `https://mattermost.atlassian.net/projects/${projectKey}?selectedItem=com.atlassian.plugins.atlassian-connect-plugin:com.kanoah.test-manager__main-project-page#!/v2/testCycle/${testCycleKey}`;
+
+    const summary = core.summary
+        .addHeading("mmctl: E2E Test Report", 2)
+        .addHeading("JUnit report summary", 3)
+        .addTable([
+            ["Total tests", `${junitStats.totalTests}`],
+            ["Passed", `${junitStats.totalPassed}`],
+            ["Failed", `${junitStats.totalFailures}`],
+            ["Skipped", `${junitStats.totalSkipped}`],
+            ["Error", `${junitStats.totalErrors}`],
+            [
+                "Duration",
+                `${junitStats.totalTime.toFixed(1)}s (~${timeInMinutes}m)`,
+            ],
+        ])
+        .addHeading("Extracted MM-T test cases", 3)
+        .addTable([
+            ["Total tests found", `${testKeyStats.totalOccurrences}`],
+            ["Unique test keys", `${testKeyStats.uniqueCount}`],
+            ["Passed", `${testKeyStats.passedCount} test keys`],
+            ["Failed", `${testKeyStats.failedCount} test keys`],
+            ["Skipped", `${testKeyStats.skippedCount} test keys`],
+        ])
+        .addHeading("Zephyr Scale Results", 3)
+        .addTable([
+            ["Test cycle key", `${testCycleKey}`],
+            ["Test cycle name", `${testCycle.name}`],
+            [
+                "Successfully saved",
+                `${successCount} executions (${uniqueSavedTestKeys.size} unique test keys)`,
+            ],
+
+            ...(failureCount === 0
+                ? []
+                : [
+                      [
+                          "Failed on saving",
+                          `${failureCount} executions (${uniqueFailedTestKeys.size} unique test keys)`,
+                      ],
+                  ]),
+        ])
+        .addLink("View in Zephyr", zephyrUrl);
+
+    await summary.write();
+}
+
+// Create Zephyr API client implementation
+export function createZephyrApiClient(apiKey: string): ZephyrApiClient {
+    return {
+        async createTestCycle(testCycle: TestCycle): Promise<{ key: string }> {
+            const response = await fetch(`${zephyrCloudApiUrl}/testcycles`, {
+                method: "POST",
+                headers: {
+                    Authorization: `Bearer ${apiKey}`,
+                    "Content-Type": "application/json; charset=utf-8",
+                },
+                body: JSON.stringify(testCycle),
+            });
+
+            if (!response.ok) {
+                const errorDetails = await response.json();
+                throw new Error(
+                    `Failed to create test cycle: ${JSON.stringify(errorDetails)} (Status: ${response.status})`,
+                );
+            }
+
+            return await response.json();
+        },
+
+        async saveTestExecution(
+            testExecution: TestExecution,
+            retries = 3,
+        ): Promise<void> {
+            for (let attempt = 1; attempt <= retries; attempt++) {
+                try {
+                    const response = await fetch(
+                        `${zephyrCloudApiUrl}/testexecutions`,
+                        {
+                            method: "POST",
+                            headers: {
+                                Authorization: `Bearer ${apiKey}`,
+                                "Content-Type":
+                                    "application/json; charset=utf-8",
+                            },
+                            body: JSON.stringify(testExecution),
+                        },
+                    );
+
+                    if (!response.ok) {
+                        const errorBody = await response.text();
+                        throw new Error(
+                            `HTTP ${response.status}: ${errorBody}`,
+                        );
+                    }
+
+                    const responseData = await response.json();
+                    core.info(
+                        `Saved test execution: ${testExecution.testCaseKey} (${testExecution.statusName}) - Response: ${JSON.stringify(responseData)}`,
+                    );
+                    return; // Success
+                } catch (error) {
+                    const errorMsg =
+                        error instanceof Error ? error.message : String(error);
+                    core.warning(
+                        `Error saving test execution for ${testExecution.testCaseKey} (attempt ${attempt}/${retries}): ${errorMsg}`,
+                    );
+
+                    if (attempt === retries) {
+                        throw new Error(
+                            `Failed after ${retries} attempts: ${errorMsg}`,
+                        );
+                    }
+
+                    // Wait before retry (exponential backoff)
+                    const delay = 1000 * attempt;
+                    core.info(
+                        `Retrying ${testExecution.testCaseKey} in ${delay}ms...`,
+                    );
+                    await new Promise((resolve) => setTimeout(resolve, delay));
+                }
+            }
+        },
+    };
+}
+
+export async function run(): Promise<void> {
+    // GitHub environment variables
+    const branch =
+        process.env.GITHUB_HEAD_REF || process.env.GITHUB_REF_NAME || "unknown";
+    const githubRepository = process.env.GITHUB_REPOSITORY || "";
+    const githubRunId = process.env.GITHUB_RUN_ID || "";
+    const githubRunUrl =
+        githubRepository && githubRunId
+            ? `https://github.com/${githubRepository}/actions/runs/${githubRunId}`
+            : "";
+
+    // Generate build number from GitHub environment variables
+    const buildNumber = `${branch}-${githubRunId}`;
+
+    // Required inputs
+    const reportPath = core.getInput("report-path", { required: true });
+    const buildImage = core.getInput("build-image", { required: true });
+    const zephyrApiKey = core.getInput("zephyr-api-key", { required: true });
+
+    // Optional inputs with defaults
+    const zephyrFolderId = parseInt(
+        core.getInput("zephyr-folder-id") || "27504432",
+        10,
+    );
+    const projectKey = core.getInput("jira-project-key") || "MM";
+
+    // Validate required fields
+    if (!reportPath) {
+        throw new Error("report-path is required");
+    }
+    if (!buildImage) {
+        throw new Error("build-image is required");
+    }
+    if (!zephyrApiKey) {
+        throw new Error("zephyr-api-key is required");
+    }
+
+    core.info(`Reading report file from: ${reportPath}`);
+    core.info(`  - Branch: ${branch}`);
+    core.info(`  - Build Image: ${buildImage}`);
+    core.info(`  - Build Number: ${buildNumber}`);
+
+    const { testCycle, testExecutions, junitStats, testKeyStats } =
+        await getTestData(reportPath, {
+            projectKey,
+            zephyrFolderId,
+            branch,
+            buildImage,
+            buildNumber,
+            githubRunUrl,
+        });
+
+    const client = createZephyrApiClient(zephyrApiKey);
+
+    core.startGroup("Creating test cycle and saving test executions in Zephyr");
+
+    // Create test cycle
+    const createdTestCycle = await client.createTestCycle(testCycle);
+    core.info(`Created test cycle: ${createdTestCycle.key}`);
+
+    // Sort and save test executions
+    const sortedExecutions = sortTestExecutions(testExecutions);
+
+    const promises = sortedExecutions.map((testExecution) => {
+        // Add project key and test cycle key
+        testExecution.projectKey = projectKey;
+        testExecution.testCycleKey = createdTestCycle.key;
+
+        return client
+            .saveTestExecution(testExecution)
+            .then(() => ({
+                success: true,
+                testCaseKey: testExecution.testCaseKey,
+            }))
+            .catch((error) => ({
+                success: false,
+                testCaseKey: testExecution.testCaseKey,
+                error: error.message,
+            }));
+    });
+    const results = await Promise.all(promises);
+    core.endGroup();
+
+    let successCount = 0;
+    let failureCount = 0;
+    const savedTestKeys: string[] = [];
+    const failedTestKeys: string[] = [];
+
+    results.forEach((result) => {
+        if (result.success) {
+            successCount++;
+            savedTestKeys.push(result.testCaseKey);
+        } else {
+            failureCount++;
+            failedTestKeys.push(result.testCaseKey);
+            const error = "error" in result ? result.error : "Unknown error";
+            core.warning(
+                `Test execution failed for ${result.testCaseKey}: ${error}`,
+            );
+        }
+    });
+
+    // Calculate unique test keys
+    const uniqueSavedTestKeys = new Set(savedTestKeys);
+    const uniqueFailedTestKeys = new Set(failedTestKeys);
+
+    // Create GitHub Actions summary (only if running in GitHub Actions environment)
+    if (process.env.GITHUB_STEP_SUMMARY) {
+        await writeGitHubSummary(
+            testCycle,
+            junitStats,
+            testKeyStats,
+            successCount,
+            failureCount,
+            uniqueSavedTestKeys,
+            uniqueFailedTestKeys,
+            projectKey,
+            createdTestCycle.key,
+        );
+    }
+
+    core.startGroup("Zephyr Scale Results");
+    core.info(`Test cycle key: ${createdTestCycle.key}`);
+    core.info(`Test cycle name: ${testCycle.name}`);
+    core.info(
+        `Successfully saved: ${successCount} executions (${uniqueSavedTestKeys.size} unique test keys)`,
+    );
+    if (failureCount > 0) {
+        core.info(
+            `Failed to save: ${failureCount} executions (${uniqueFailedTestKeys.size} unique test keys)`,
+        );
+    }
+    core.info(
+        `View in Zephyr: https://mattermost.atlassian.net/projects/${projectKey}?selectedItem=com.atlassian.plugins.atlassian-connect-plugin:com.kanoah.test-manager__main-project-page#!/v2/testCycle/${createdTestCycle.key}`,
+    );
+
+    if (failedTestKeys.length > 0) {
+        core.info(
+            `Failed test keys (${failedTestKeys.length} total, ${uniqueFailedTestKeys.size} unique): ${failedTestKeys.join(", ")}`,
+        );
+    }
+    core.endGroup();
+
+    // JUnit summary outputs
+    core.setOutput("junit-total-tests", junitStats.totalTests);
+    core.setOutput("junit-total-passed", junitStats.totalPassed);
+    core.setOutput("junit-total-failed", junitStats.totalFailures);
+    core.setOutput("junit-pass-rate", junitStats.passRate);
+    core.setOutput("junit-duration-seconds", junitStats.totalTime.toFixed(1));
+
+    // Zephyr results outputs
+    core.setOutput("test-cycle", createdTestCycle.key);
+    core.setOutput("test-keys-execution-count", testExecutions.length);
+    core.setOutput("test-keys-unique-count", uniqueSavedTestKeys.size);
+}
@@ -0,0 +1,50 @@
+export interface TestExecution {
+    testCaseKey: string;
+    statusName: string;
+    executionTime: number;
+    comment: string;
+    projectKey?: string;
+    testCycleKey?: string;
+}
+
+export interface TestCycle {
+    projectKey: string;
+    name: string;
+    description: string;
+    statusName: string;
+    folderId: number;
+    plannedStartDate?: string;
+    plannedEndDate?: string;
+    customFields?: Record<string, any>;
+}
+
+export interface TestData {
+    testCycle: TestCycle;
+    testExecutions: TestExecution[];
+    junitStats: {
+        totalTests: number;
+        totalFailures: number;
+        totalErrors: number;
+        totalSkipped: number;
+        totalPassed: number;
+        passRate: string;
+        totalTime: number;
+    };
+    testKeyStats: {
+        totalOccurrences: number;
+        uniqueCount: number;
+        passedCount: number;
+        failedCount: number;
+        skippedCount: number;
+        failedKeys: string[];
+        skippedKeys: string[];
+    };
+}
+
+export interface ZephyrApiClient {
+    createTestCycle(testCycle: TestCycle): Promise<{ key: string }>;
+    saveTestExecution(
+        testExecution: TestExecution,
+        retries?: number,
+    ): Promise<void>;
+}
@@ -0,0 +1,13 @@
+{
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "commonjs",
+    "outDir": "./lib",
+    "rootDir": "./src",
+    "strict": true,
+    "noImplicitAny": true,
+    "esModuleInterop": true,
+    "typeRoots": ["./node_modules/@types"]
+  },
+  "exclude": ["node_modules", "../../../node_modules"]
+}
@@ -0,0 +1,12 @@
+import { defineConfig } from 'tsup';
+
+export default defineConfig({
+  entry: ['src/index.ts'],
+  format: ['cjs'],
+  outDir: 'dist',
+  clean: true,
+  noExternal: [/.*/], // Bundle all dependencies
+  minify: false,
+  sourcemap: false,
+  target: 'node24',
+});
@@ -94,6 +94,14 @@ jobs:
          cd server/build
          docker compose --ansi never stop

+      - name: Save mmctl test report to Zephyr Scale
+        if: ${{ always() && hashFiles('server/report.xml') != '' && github.event_name != 'pull_request' && (github.ref_name == 'master' || startsWith(github.ref_name, 'release-')) }}
+        uses: ./.github/actions/save-junit-report-tms
+        with:
+          report-path: server/report.xml
+          zephyr-api-key: ${{ secrets.MM_E2E_ZEPHYR_API_KEY }}
+          build-image: ${{ steps.build.outputs.BUILD_IMAGE }}
+
      - name: Archive logs
        if: ${{ always() }}
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2