Add --stop-on-error option for Docker wheel testing

Adds a new CLI option to control test execution behavior when failures occur:
- Without --stop-on-error (default): Continue running all tests even if some fail
- With --stop-on-error: Stop execution on first test failure (previous behavior)

This enables comprehensive debugging by running both opm-common and omp-simulators
tests in a single run, showing all failures at once rather than stopping at the first one.

Changes:
- Add --stop-on-error flag to run-docker-image command
- Pass STOP_ON_ERROR environment variable to Docker container
- Update entrypoint.py to handle continue-on-error behavior
- Modify test functions to check return codes and continue when appropriate
- Maintain backward compatibility with existing error handling

python/docker/test_wheels/dockerfiles/common/entrypoint.py

@@ -13,6 +13,10 @@ import shutil
 import subprocess
 import sys
 
+def should_stop_on_error() -> bool:
+    """Check if we should stop on test errors based on environment variable."""
+    return os.environ.get("STOP_ON_ERROR") == "1"
+
 def copy_essential_directories(source_dir: str, dest_dir: str, essential_items: list) -> None:
     """Copy essential directories from source to destination, avoiding problematic items.
 
@@ -33,10 +37,11 @@ def copy_essential_directories(source_dir: str, dest_dir: str, essential_items:
             else:
                 shutil.copy2(item_path, dest_path)
 
-def run_opm_common_tests(python_bin: str) -> None:
+def run_opm_common_tests(python_bin: str) -> bool:
     """Run all unittests in opm-common/python/tests"""
     # Check for specific test selection
     test_cases = os.environ.get("TEST_CASES_COMMON")
+    stop_on_error = should_stop_on_error()
 
     if os.environ.get("USE_HOST_TESTS"):
         # Copy strategy for host-mounted directories:
@@ -54,43 +59,74 @@ def run_opm_common_tests(python_bin: str) -> None:
         # Run tests from clean location (preserves package structure for relative imports)
         if test_cases:
             # Run specific test files
+            all_passed = True
             for test_case in test_cases.split(","):
                 test_case = test_case.strip()
                 print(f"Running opm-common test: {test_case}")
-                subprocess.run([
+                result = subprocess.run([
                     python_bin, "-m", "unittest", f"tests.test_{test_case}",
                     "-v"
-                ], cwd=test_dir, check=True)
+                ], cwd=test_dir, check=False)
+                if result.returncode != 0:
+                    print(f"Test {test_case} failed with return code {result.returncode}", file=sys.stderr)
+                    all_passed = False
+                    if stop_on_error:
+                        raise subprocess.CalledProcessError(result.returncode, result.args)
+            if not all_passed:
+                print("Some opm-common specific tests failed", file=sys.stderr)
+            return all_passed
         else:
             # Run all tests (original behavior)
-            subprocess.run([
+            result = subprocess.run([
                 python_bin, "-m", "unittest", "discover",
                 "-s", test_dir,
                 "-p", "test_*.py",
                 "-v"
-            ], cwd=test_dir, check=True)
+            ], cwd=test_dir, check=False)
+            if result.returncode != 0:
+                print(f"opm-common tests failed with return code {result.returncode}", file=sys.stderr)
+                if stop_on_error:
+                    raise subprocess.CalledProcessError(result.returncode, result.args)
+                return False
+            return True
     else:
         if test_cases:
             # Run specific test files in cloned repo environment
+            all_passed = True
             for test_case in test_cases.split(","):
                 test_case = test_case.strip()
                 print(f"Running opm-common test: {test_case}")
-                subprocess.run([
+                result = subprocess.run([
                     python_bin, "-m", "unittest", f"test_{test_case}",
                     "-v"
-                ], cwd="/test/opm-common/python", check=True)
+                ], cwd="/test/opm-common/python", check=False)
+                if result.returncode != 0:
+                    print(f"Test {test_case} failed with return code {result.returncode}", file=sys.stderr)
+                    all_passed = False
+                    if stop_on_error:
+                        raise subprocess.CalledProcessError(result.returncode, result.args)
+            if not all_passed:
+                print("Some opm-common specific tests failed", file=sys.stderr)
+            return all_passed
         else:
             # Original strategy: run from cloned repos (opm directory was moved to opm_original during build)
-            subprocess.run([
+            result = subprocess.run([
                 python_bin, "-m", "unittest", "discover",
                 "-s", "/test/opm-common/python",  # path to the tests
                 "-v"  # verbose output
-            ], check=True)
+            ], check=False)
+            if result.returncode != 0:
+                print(f"opm-common tests failed with return code {result.returncode}", file=sys.stderr)
+                if stop_on_error:
+                    raise subprocess.CalledProcessError(result.returncode, result.args)
+                return False
+            return True
 
-def run_opm_simulators_tests(python_bin: str) -> None:
+def run_opm_simulators_tests(python_bin: str) -> bool:
     """Run all unittests in opm-simulators/python/test"""
     # Check for specific test selection
     test_cases = os.environ.get("TEST_CASES_SIMULATORS")
+    stop_on_error = should_stop_on_error()
 
     if os.environ.get("USE_HOST_TESTS"):
         # Copy strategy for host-mounted directories:
@@ -105,38 +141,68 @@ def run_opm_simulators_tests(python_bin: str) -> None:
         # Use parent directory as start and specify test pattern to make relative imports work
         if test_cases:
             # Run specific test files
+            all_passed = True
             for test_case in test_cases.split(","):
                 test_case = test_case.strip()
                 print(f"Running opm-simulators test: {test_case}")
-                subprocess.run([
+                result = subprocess.run([
                     python_bin, "-m", "unittest", f"test.test_{test_case}",
                     "-v"
-                ], cwd=test_dir, check=True)
+                ], cwd=test_dir, check=False)
+                if result.returncode != 0:
+                    print(f"Test {test_case} failed with return code {result.returncode}", file=sys.stderr)
+                    all_passed = False
+                    if stop_on_error:
+                        raise subprocess.CalledProcessError(result.returncode, result.args)
+            if not all_passed:
+                print("Some opm-simulators specific tests failed", file=sys.stderr)
+            return all_passed
         else:
             # Run all tests (original behavior)
-            subprocess.run([
+            result = subprocess.run([
                 python_bin, "-m", "unittest", "discover",
                 "-s", test_dir,
                 "-p", "test_*.py",
                 "-v"
-            ], cwd=test_dir, check=True)
+            ], cwd=test_dir, check=False)
+            if result.returncode != 0:
+                print(f"opm-simulators tests failed with return code {result.returncode}", file=sys.stderr)
+                if stop_on_error:
+                    raise subprocess.CalledProcessError(result.returncode, result.args)
+                return False
+            return True
     else:
         if test_cases:
             # Run specific test files in cloned repo environment
+            all_passed = True
             for test_case in test_cases.split(","):
                 test_case = test_case.strip()
                 print(f"Running opm-simulators test: {test_case}")
-                subprocess.run([
+                result = subprocess.run([
                     python_bin, "-m", "unittest", f"test.test_{test_case}",
                     "-v"
-                ], cwd="/test/opm-simulators/python", check=True)
+                ], cwd="/test/opm-simulators/python", check=False)
+                if result.returncode != 0:
+                    print(f"Test {test_case} failed with return code {result.returncode}", file=sys.stderr)
+                    all_passed = False
+                    if stop_on_error:
+                        raise subprocess.CalledProcessError(result.returncode, result.args)
+            if not all_passed:
+                print("Some opm-simulators specific tests failed", file=sys.stderr)
+            return all_passed
         else:
             # Original strategy: run from cloned repos (opm directory was moved to opm_original during build)
-            subprocess.run([
+            result = subprocess.run([
                 python_bin, "-m", "unittest", "discover",
                 "-s", "/test/opm-simulators/python",  # path to the tests
                 "-v"  # verbose output
-            ], check=True)
+            ], check=False)
+            if result.returncode != 0:
+                print(f"opm-simulators tests failed with return code {result.returncode}", file=sys.stderr)
+                if stop_on_error:
+                    raise subprocess.CalledProcessError(result.returncode, result.args)
+                return False
+            return True
 
 
 def install_and_test(python_version, pyenv_full_version):
@@ -188,20 +254,35 @@ def install_and_test(python_version, pyenv_full_version):
         test_cases_common = os.environ.get("TEST_CASES_COMMON")
         test_cases_simulators = os.environ.get("TEST_CASES_SIMULATORS")
 
+        # Track overall test success
+        overall_success = True
+
         # If no specific test cases are specified, run all tests (original behavior)
         if not test_cases_common and not test_cases_simulators:
-            run_opm_simulators_tests(python_bin)
-            run_opm_common_tests(python_bin)
+            simulators_success = run_opm_simulators_tests(python_bin)
+            common_success = run_opm_common_tests(python_bin)
+            overall_success = simulators_success and common_success
         else:
             # Run only the requested test suites
             if test_cases_simulators or not test_cases_common:
                 # Run simulators tests if explicitly requested OR if common tests not specified
-                run_opm_simulators_tests(python_bin)
+                simulators_success = run_opm_simulators_tests(python_bin)
+                overall_success = overall_success and simulators_success
             if test_cases_common or not test_cases_simulators:
                 # Run common tests if explicitly requested OR if simulators tests not specified
-                run_opm_common_tests(python_bin)
+                common_success = run_opm_common_tests(python_bin)
+                overall_success = overall_success and common_success
 
-        return True  # All good!
+        # Report overall result
+        if overall_success:
+            return True
+        elif should_stop_on_error():
+            # When stop_on_error is True, failures should stop execution
+            return False
+        else:
+            # When stop_on_error is False, continue even if tests failed
+            print(f"Tests failed for Python {python_version}, but continuing due to continue-on-error mode", file=sys.stderr)
+            return True
 
     except subprocess.CalledProcessError:
         print(f"Installation or test failed for Python {python_version}", file=sys.stderr)

python/docker/test_wheels/scripts/src/opm_wheels/helpers.py

@@ -132,7 +132,8 @@ def run_docker_run(
     python_versions: list[PythonVersion],
     host_tests_dir: Path = None,
     test_cases_common: str = None,
-    test_cases_simulators: str = None
+    test_cases_simulators: str = None,
+    stop_on_error: bool = False
 ) -> None:
     # Build Docker run command
     docker_cmd = [
@@ -173,6 +174,13 @@ def run_docker_run(
         docker_cmd.extend(["-e", f"TEST_CASES_SIMULATORS={test_cases_simulators}"])
         logging.info(f"Running only opm-simulators tests: {test_cases_simulators}")
 
+    # Add stop on error environment variable
+    if stop_on_error:
+        docker_cmd.extend(["-e", "STOP_ON_ERROR=1"])
+        logging.info("Running with stop-on-error enabled")
+    else:
+        logging.info("Running with continue-on-error (default behavior)")
+
     # Add the Docker image tag
     docker_cmd.append(docker_tag)
 

python/docker/test_wheels/scripts/src/opm_wheels/main.py

@@ -161,6 +161,12 @@ def build_docker_image(
     help="Comma-separated list of specific test case patterns to run for opm-simulators (e.g., 'basic,mpi'). "
          "Will run test_<pattern>.py files. If not specified, all tests are run."
 )
+@click.option(
+    "--stop-on-error",
+    is_flag=True,
+    default=False,
+    help="Stop execution on first test failure. If not specified, all tests will run even if some fail."
+)
 def run_docker_image(
     docker_os: str,
     wheel_dir: str,
@@ -168,6 +174,7 @@ def run_docker_image(
     host_tests_dir: Path = None,
     test_cases_common: str = None,
     test_cases_simulators: str = None,
+    stop_on_error: bool = False,
 ) -> None:
     f"""Run the Docker image. The option "--docker-tag" specifies the tag of the Docker
     image to run. The option "--python-versions" specifies the Python versions to
@@ -177,4 +184,4 @@ def run_docker_image(
     wheel_dir = helpers.get_wheel_abs_dir(wheel_dir)
     canonical_os = helpers.canonicalize_docker_os(docker_os)
     docker_tag = helpers.get_docker_tag(canonical_os)
-    helpers.run_docker_run(docker_tag, wheel_dir, python_versions, host_tests_dir, test_cases_common, test_cases_simulators)
+    helpers.run_docker_run(docker_tag, wheel_dir, python_versions, host_tests_dir, test_cases_common, test_cases_simulators, stop_on_error)