Add option to use opm python tests folders from host

This will help avoid rebuilding the test containers each time
the host data changes.
This commit is contained in:
Håkon Hægland committed 2025-09-04 13:25:57 +02:00
1 parent ec56668d8b
commit 9167a35ae4
3 files changed
+106 -10

No files matched your search

+64 -1
View File
@@ -92,6 +92,12 @@ $ opm-wheels run-docker-image \
--docker-os="debian:11" \
--wheel-dir="/path/to/wheelhouse-shared" \
--python-versions="3.12"
# Test using host test directories (CI/CD friendly)
$ opm-wheels run-docker-image \
--docker-os="ubuntu:24.04" \
--wheel-dir="/path/to/wheelhouse" \
--host-tests-dir="/path/to/opm-repos"
```
### Available options for `run-docker-image`:
@@ -100,6 +106,54 @@ $ opm-wheels run-docker-image \
- `--python-versions`, `-p` - Python versions for testing
- Supported: `3.8,3.9,3.10,3.11,3.12,3.13`
- Format: comma-separated list without spaces
- `--host-tests-dir` - Use test directories from host instead of cloned repos
- Expects: `host-tests-dir/opm-common/python` and `host-tests-dir/opm-simulators/python`
- Eliminates need for repository cloning and nightly container rebuilds
## CI/CD Usage
### Jenkins Pipeline Example
For Jenkins environments with network restrictions, use `--host-tests-dir` to avoid repository cloning:
```bash
#!/bin/bash
# Jenkins script - assumes workspace contains checked-out opm repositories
# Build test container once (can be cached/reused)
opm-wheels build-docker-image --docker-os="ubuntu:24.04"
# Run tests using host directories (no network access needed)
opm-wheels run-docker-image \
--docker-os="ubuntu:24.04" \
--wheel-dir="$WORKSPACE/wheels" \
--host-tests-dir="$WORKSPACE" \
--python-versions="3.11,3.12,3.13"
```
### Directory Structure for Host Tests
When using `--host-tests-dir`, ensure your directory structure matches:
```
your-repos-dir/
├── opm-common/
│ └── python/ # Contains Python test files
│ ├── tests/
│ └── test_*.py
└── opm-simulators/
└── python/ # Contains Python test files
├── tests/
└── test_*.py
```
### Benefits for CI/CD
- **No nightly rebuilds**: Container image can be built once and reused
- **Network isolation**: No git clones during test execution
- **Version consistency**: Tests run against exact workspace codebase
- **Faster execution**: Skip repository cloning step
- **Debugging friendly**: Test failures map directly to workspace files
## Additional commands
@@ -142,6 +196,15 @@ This error occurs when the CLI cannot find the configuration file. Solutions:
### RuntimeError: Not a valid Git repository
This error means you're not inside a Git repository. Either:
- Navigate to the opm-simulators repository directory, or
- Navigate to the opm-simulators repository directory, or
- Use the `OPM_SIMULATORS_ROOT` environment variable method
### FileNotFoundError: Expected opm-*/python directory not found
When using `--host-tests-dir`, this error means the directory structure is incorrect. Ensure your directory contains:
```
host-tests-dir/
├── opm-common/python/ # Must exist with test files
└── omp-simulators/python/ # Must exist with test files
```
@@ -102,17 +102,42 @@ def run_docker_build(
def run_docker_run(
docker_tag: str,
wheel_dir: Path,
python_versions: list[PythonVersion]
python_versions: list[PythonVersion],
host_tests_dir: Path = None
) -> None:
# Build Docker run command
docker_cmd = [
"docker", "run",
"--rm",
"-v", str(wheel_dir) + ":/test/wheels/",
"-e", "PYTHON_VERSIONS=" + ",".join([str(v) for v in python_versions])
]
# Add host test directory mounts if specified
if host_tests_dir:
# Validate that the expected directories exist
opm_common_python = host_tests_dir / "opm-common" / "python"
opm_simulators_python = host_tests_dir / "opm-simulators" / "python"
if not opm_common_python.exists():
raise FileNotFoundError(f"Expected opm-common/python directory not found at: {opm_common_python}")
if not opm_simulators_python.exists():
raise FileNotFoundError(f"Expected opm-simulators/python directory not found at: {opm_simulators_python}")
# Add volume mounts for test directories
docker_cmd.extend([
"-v", str(opm_common_python) + ":/test/opm-common/python",
"-v", str(opm_simulators_python) + ":/test/opm-simulators/python"
])
logging.info(f"Using host test directories from: {host_tests_dir}")
# Add the Docker image tag
docker_cmd.append(docker_tag)
try:
result = subprocess.run(
[
"docker", "run",
"--rm",
"-v", str(wheel_dir) + ":/test/wheels/",
"-e", "PYTHON_VERSIONS=" + ",".join([str(v) for v in python_versions]),
docker_tag
],
docker_cmd,
check=True, # Raise an exception if the process returns a non-zero exit code
)
logging.info(f"Docker image with tag {docker_tag} run successfully.")
@@ -120,10 +120,18 @@ def build_docker_image(
help="The Python versions to use for testing. Valid values are: "
f"{', '.join(PythonVersion.valid_versions())}"
)
@click.option(
"--host-tests-dir",
type=click.Path(exists=True, file_okay=False, dir_okay=True, path_type=Path),
help="Directory containing opm-common and opm-simulators repositories from host. "
"If specified, tests will be run from host directories instead of cloned repos. "
"Expected structure: host-tests-dir/opm-common/python and host-tests-dir/opm-simulators/python"
)
def run_docker_image(
docker_os: str,
wheel_dir: str,
python_versions: list[PythonVersion],
host_tests_dir: Path = None,
) -> None:
f"""Run the Docker image. The option "--docker-tag" specifies the tag of the Docker
image to run. The option "--python-versions" specifies the Python versions to
@@ -132,4 +140,4 @@ def run_docker_image(
If not given, the default value is {','.join(get_default_versions())}."""
wheel_dir = helpers.get_wheel_abs_dir(wheel_dir)
docker_tag = helpers.get_docker_tag(docker_os)
helpers.run_docker_run(docker_tag, wheel_dir, python_versions)
helpers.run_docker_run(docker_tag, wheel_dir, python_versions, host_tests_dir)