From ddf2e3d3385f5cdd264e31ebda3d7ab719c44cc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?H=C3=A5kon=20H=C3=A6gland?= Date: Tue, 23 Sep 2025 06:57:05 +0200 Subject: [PATCH] Add Python version selection to Docker image builds This enhancement allows specifying which Python versions to install during Docker image builds, significantly reducing build times when only testing specific Python versions during debugging. Changes: - Add --python-versions option to build-docker-image command - Pass BUILD_PYTHON_VERSIONS build argument to Docker - Update install_python_versions.py to respect BUILD_PYTHON_VERSIONS env var - Validate requested versions against supported versions in python_versions.json - Maintain backward compatibility (installs all versions if not specified) Usage examples: # Build with only Python 3.12 (much faster for debugging) opm-wheels build-docker-image --docker-os="ubuntu-24.04" --python-versions="3.12" # Build with multiple specific versions omp-wheels build-docker-image --python-versions="3.11,3.12" # Build with all versions (original behavior) opm-wheels build-docker-image Benefits: - Reduces build time from ~7 minutes to ~2 minutes for single Python version - Enables faster iteration during debugging of Docker-specific issues - Particularly valuable when combined with --test-cases-* for focused testing --- .../common/install_python_versions.py | 21 +++++++++- .../dockerfiles/debian-11/Dockerfile | 4 ++ .../dockerfiles/ubuntu-22.04/Dockerfile | 4 ++ .../dockerfiles/ubuntu-24.04/Dockerfile | 4 ++ .../scripts/src/opm_wheels/helpers.py | 39 ++++++++++++------- .../scripts/src/opm_wheels/main.py | 13 ++++++- 6 files changed, 67 insertions(+), 18 deletions(-) diff --git a/python/docker/test_wheels/dockerfiles/common/install_python_versions.py b/python/docker/test_wheels/dockerfiles/common/install_python_versions.py index 3f9dae2698..7ecf924065 100755 --- a/python/docker/test_wheels/dockerfiles/common/install_python_versions.py +++ b/python/docker/test_wheels/dockerfiles/common/install_python_versions.py @@ -9,8 +9,25 @@ try: config_file = "/test/python_versions.json" with open(config_file) as f: config = json.load(f) - version_map = config["supported_versions"] - print(f"📋 Using Python versions from python_versions.json: {list(version_map.keys())}") + full_version_map = config["supported_versions"] + + # Check if specific Python versions were requested via build argument + build_versions = os.environ.get("BUILD_PYTHON_VERSIONS") + if build_versions: + requested_versions = [v.strip() for v in build_versions.split(",")] + version_map = {v: full_version_map[v] for v in requested_versions if v in full_version_map} + print(f"📋 Using requested Python versions: {list(version_map.keys())}") + + # Validate that all requested versions are supported + missing_versions = [v for v in requested_versions if v not in full_version_map] + if missing_versions: + print(f"Error: Unsupported Python version(s): {missing_versions}") + print(f"Supported versions: {list(full_version_map.keys())}") + exit(1) + else: + version_map = full_version_map + print(f"📋 Using all Python versions from python_versions.json: {list(version_map.keys())}") + except FileNotFoundError: print("Error: python_versions.json not found!") exit(1) diff --git a/python/docker/test_wheels/dockerfiles/debian-11/Dockerfile b/python/docker/test_wheels/dockerfiles/debian-11/Dockerfile index 6cf78f32b7..9f8b8e3ae6 100644 --- a/python/docker/test_wheels/dockerfiles/debian-11/Dockerfile +++ b/python/docker/test_wheels/dockerfiles/debian-11/Dockerfile @@ -21,6 +21,10 @@ RUN /test/common/setup_pyenv.sh # Switch to bash for pyenv usage SHELL ["/bin/bash", "-c"] +# Accept Python versions build argument and set as environment variable +ARG BUILD_PYTHON_VERSIONS +ENV BUILD_PYTHON_VERSIONS=${BUILD_PYTHON_VERSIONS} + RUN /usr/bin/python3 /test/common/install_python_versions.py # Clone opm-common and opm-simulators to get the unit tests diff --git a/python/docker/test_wheels/dockerfiles/ubuntu-22.04/Dockerfile b/python/docker/test_wheels/dockerfiles/ubuntu-22.04/Dockerfile index 848cb69515..6a921dbe11 100644 --- a/python/docker/test_wheels/dockerfiles/ubuntu-22.04/Dockerfile +++ b/python/docker/test_wheels/dockerfiles/ubuntu-22.04/Dockerfile @@ -21,6 +21,10 @@ RUN /test/common/setup_pyenv.sh # Switch to bash for pyenv usage SHELL ["/bin/bash", "-c"] +# Accept Python versions build argument and set as environment variable +ARG BUILD_PYTHON_VERSIONS +ENV BUILD_PYTHON_VERSIONS=${BUILD_PYTHON_VERSIONS} + RUN /usr/bin/python3 /test/common/install_python_versions.py # Clone opm-common and opm-simulators to get the unit tests diff --git a/python/docker/test_wheels/dockerfiles/ubuntu-24.04/Dockerfile b/python/docker/test_wheels/dockerfiles/ubuntu-24.04/Dockerfile index d02b617f1d..7bbd3c1b9b 100644 --- a/python/docker/test_wheels/dockerfiles/ubuntu-24.04/Dockerfile +++ b/python/docker/test_wheels/dockerfiles/ubuntu-24.04/Dockerfile @@ -21,6 +21,10 @@ RUN /test/common/setup_pyenv.sh # Switch to bash for pyenv usage SHELL ["/bin/bash", "-c"] +# Accept Python versions build argument and set as environment variable +ARG BUILD_PYTHON_VERSIONS +ENV BUILD_PYTHON_VERSIONS=${BUILD_PYTHON_VERSIONS} + RUN /usr/bin/python3 /test/common/install_python_versions.py # Clone opm-common and opm-simulators to get the unit tests diff --git a/python/docker/test_wheels/scripts/src/opm_wheels/helpers.py b/python/docker/test_wheels/scripts/src/opm_wheels/helpers.py index 5a6c30dde5..a98d3a9e6b 100644 --- a/python/docker/test_wheels/scripts/src/opm_wheels/helpers.py +++ b/python/docker/test_wheels/scripts/src/opm_wheels/helpers.py @@ -89,26 +89,37 @@ def run_docker_build( build_ctxt_dir: Path, docker_file: Path, docker_progress: bool = False, + python_versions: list[PythonVersion] = None, ) -> None: if docker_progress: progress="plain" else: progress="auto" + # Build Docker command with base arguments + docker_cmd = [ + "docker", "build", + "--progress", progress, + "--build-arg", "opm_simulators_repo=" + opm_simulators_repo, + "--build-arg", "opm_common_repo=" + opm_common_repo, + "--build-arg", "opm_simulators_branch=" + opm_simulators_branch, + "--build-arg", "opm_common_branch=" + opm_common_branch, + ] + + # Add Python versions build argument if specified + if python_versions: + python_versions_str = ",".join([str(v) for v in python_versions]) + docker_cmd.extend(["--build-arg", f"BUILD_PYTHON_VERSIONS={python_versions_str}"]) + logging.info(f"Building Docker image with Python versions: {python_versions_str}") + + # Add tag and context + docker_cmd.extend([ + "-t", docker_tag, + "-f", str(docker_file), + str(build_ctxt_dir) + ]) + try: - result = subprocess.run( - [ - "docker", "build", - "--progress", progress, - "--build-arg", "opm_simulators_repo=" + opm_simulators_repo, - "--build-arg", "opm_common_repo=" + opm_common_repo, - "--build-arg", "opm_simulators_branch=" + opm_simulators_branch, - "--build-arg", "opm_common_branch=" + opm_common_branch, - "-t", docker_tag, - "-f", str(docker_file), - str(build_ctxt_dir) - ], - check=True, # Raise an exception if the process returns a non-zero exit code - ) + result = subprocess.run(docker_cmd, check=True) logging.info(f"Docker image with tag {docker_tag} built successfully.") except subprocess.CalledProcessError as e: logging.error(f"Error building Docker image for tag {docker_tag}: {e}") diff --git a/python/docker/test_wheels/scripts/src/opm_wheels/main.py b/python/docker/test_wheels/scripts/src/opm_wheels/main.py index c5bd55d64c..652c75f5cc 100644 --- a/python/docker/test_wheels/scripts/src/opm_wheels/main.py +++ b/python/docker/test_wheels/scripts/src/opm_wheels/main.py @@ -77,13 +77,21 @@ def show_wheel_files(name: str, wheel_dir: str) -> None: default=False, help="Show Docker progress output." ) +@click.option( + "--python-versions", "-p", + type=str, + callback=click_helpers.validate_python_versions, + help="Python versions to install in the Docker image. Valid values are: " + f"{', '.join(PythonVersion.valid_versions())}. If not specified, all supported versions are installed." +) def build_docker_image( docker_os: str, opm_simulators_repo: str, opm_common_repo: str, opm_simulators_branch: str, opm_common_branch: str, - docker_progress: bool + docker_progress: bool, + python_versions: list[PythonVersion] = None ) -> None: """Build the Docker image. The option "--docker-os specifies the name of the OS to use in the Docker image. Supported: ubuntu:22.04, ubuntu:24.04, debian:11. If option --docker-progress @@ -112,7 +120,8 @@ def build_docker_image( opm_common_branch, build_ctxt_dir=docker_root, docker_file=dockerfile, - docker_progress=docker_progress + docker_progress=docker_progress, + python_versions=python_versions ) @main.command()