From 713e977bfc189825e367349bd146721027bc109a Mon Sep 17 00:00:00 2001 From: Magne Sjaastad Date: Sun, 24 May 2026 10:02:43 +0200 Subject: [PATCH] CI: add Ubuntu 24.04 prebuilt Docker image for Linux CI and Copilot agent Adds .github/docker/Dockerfile.ubuntu24 -- a multi-stage image bundling gcc-14, clang-19, Qt 6.7.0, the buildcache binary from the canonical GitLab repo, per-compiler vcpkg binary caches, and per-compiler warm ResInsight buildcaches. Published to ghcr.io//ci-ubuntu24 by a new nightly build workflow that mirrors the RHEL8 pattern, with a matching daily cleanup workflow that applies the same GHCR retention rules. Adds .github/workflows/ResInsightWithCacheLinuxContainer.yml -- a parallel companion to the existing Linux legs in ResInsightWithCache.yml, running both compilers in a matrix inside the container with /src as the working directory so buildcache hashes line up with the warmup baked into the image. The existing ResInsightWithCache.yml Linux legs remain unchanged during the validation phase. copilot-setup-steps.yml will be migrated to the container in a follow-up once the image is proven. --- .github/docker/Dockerfile.ubuntu24 | 284 ++++++++++++++++++ .github/docker/README.md | 178 +++++++++-- .../ResInsightWithCacheLinuxContainer.yml | 221 ++++++++++++++ .github/workflows/build-ubuntu24-image.yml | 109 +++++++ .github/workflows/cleanup-ubuntu24-image.yml | 131 ++++++++ 5 files changed, 905 insertions(+), 18 deletions(-) create mode 100644 .github/docker/Dockerfile.ubuntu24 create mode 100644 .github/workflows/ResInsightWithCacheLinuxContainer.yml create mode 100644 .github/workflows/build-ubuntu24-image.yml create mode 100644 .github/workflows/cleanup-ubuntu24-image.yml diff --git a/.github/docker/Dockerfile.ubuntu24 b/.github/docker/Dockerfile.ubuntu24 new file mode 100644 index 0000000000..f22cfe185d --- /dev/null +++ b/.github/docker/Dockerfile.ubuntu24 @@ -0,0 +1,284 @@ +# syntax=docker/dockerfile:1 + +# ============================================================================= +# Ubuntu 24.04 CI image for ResInsight. +# +# Bakes in: +# * Both toolchains used by the cache workflow's Linux legs: gcc-14 from +# Ubuntu's repos and clang-19 from the upstream LLVM apt repo. +# * Qt 6.7.0 via aqtinstall. +# * buildcache built from source (same v0.33.0 as the RHEL8 image, so the +# two images produce identical buildcache hashes for shared TUs). +# * A vcpkg binary cache per compiler at /opt/vcpkg-cache-{gcc,clang}. +# * A warm ResInsight buildcache per compiler at /opt/buildcache-{gcc,clang}, +# populated by full builds at the fixed path /src -> /src/cmakebuild so +# consumer workflows can clone into /src and get cache hits. +# +# Consumed by: +# * .github/workflows/ResInsightWithCacheLinuxContainer.yml (matrix over +# both compilers). +# * .github/workflows/copilot-setup-steps.yml (Copilot Coding Agent; +# defaults to gcc-14 via the ENV block at the bottom of this file). +# +# Rebuilt nightly by .github/workflows/build-ubuntu24-image.yml. +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Stage 1: base toolchain (both compilers + Qt + buildcache). +# ----------------------------------------------------------------------------- +FROM ubuntu:24.04 AS base + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +ENV DEBIAN_FRONTEND=noninteractive \ + LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 \ + PIP_BREAK_SYSTEM_PACKAGES=1 + +# Base development tools and Qt/GL runtime/headers. The package list mirrors +# the apt step in ResInsightWithCache.yml + copilot-setup-steps.yml, plus the +# xcb/font/x11 *-dev packages Qt 6 needs at build link time (the runner image +# happens to have most of these preinstalled; the bare ubuntu:24.04 image does +# not). +RUN apt-get update --option="APT::Acquire::Retries=3" \ + && apt-get install -y --no-install-recommends --option="APT::Acquire::Retries=3" \ + build-essential ninja-build cmake git curl wget ca-certificates \ + zip unzip tar xz-utils pkg-config gnupg lsb-release \ + flex bison autoconf automake libtool perl \ + python3.12 python3.12-venv python3-pip \ + libxkbcommon-x11-0 libxkbcommon-dev \ + libgl1-mesa-dev mesa-common-dev libegl1-mesa-dev libglu1-mesa-dev libglfw3-dev \ + libhdf5-dev \ + libxcb1-dev libxcb-keysyms1-dev libxcb-image0-dev libxcb-icccm4-dev \ + libxcb-render-util0-dev libxcb-render0-dev libxcb-shape0-dev \ + libxcb-randr0-dev libxcb-sync-dev libxcb-xfixes0-dev libxcb-xkb-dev \ + libxcb-shm0-dev libxcb-cursor0 libxkbcommon-x11-dev \ + libfontconfig1-dev libfreetype-dev \ + libx11-dev libxext-dev libxrender-dev libxi-dev libxcursor-dev \ + libdbus-1-3 libnss3 libglib2.0-0t64 \ + && rm -rf /var/lib/apt/lists/* + +# gcc-14 is in Ubuntu 24.04's repositories; no extra source needed. +RUN apt-get update --option="APT::Acquire::Retries=3" \ + && apt-get install -y --no-install-recommends --option="APT::Acquire::Retries=3" \ + gcc-14 g++-14 \ + && rm -rf /var/lib/apt/lists/* + +# clang-19 from the upstream LLVM apt repo. Mirrors the install block at +# ResInsightWithCache.yml:156-183 so the image's clang matches what the +# non-containerized leg used to install at runtime. +RUN wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \ + | gpg --dearmor -o /usr/share/keyrings/llvm-snapshot.gpg \ + && echo "deb [signed-by=/usr/share/keyrings/llvm-snapshot.gpg] http://apt.llvm.org/noble/ llvm-toolchain-noble-19 main" \ + > /etc/apt/sources.list.d/llvm-19.list \ + && apt-get update --option="APT::Acquire::Retries=3" \ + && apt-get install -y --no-install-recommends --option="APT::Acquire::Retries=3" \ + clang-19 lldb-19 lld-19 clangd-19 clang-tidy-19 clang-format-19 clang-tools-19 \ + libc++-19-dev libc++abi-19-dev libomp-19-dev \ + && rm -rf /var/lib/apt/lists/* \ + && clang-19 --version + +# Python tooling. PEP 668 marks the system Python as externally managed; for +# a single-purpose CI image --break-system-packages is the cleanest path and +# avoids a venv activation dance in every consumer workflow. grpcio-tools is +# needed at warmup time (RESINSIGHT_ENABLE_GRPC=ON). Pip itself is left at the +# system version (24.x): the Debian package has no RECORD file so pip cannot +# uninstall it for an upgrade, and the system version is recent enough to +# install everything we need. +RUN python3 -m pip install --no-cache-dir --break-system-packages \ + aqtinstall grpcio-tools + +# Qt 6.7.0 -- same version the cache workflow's Linux legs install via +# CeetronSolutions/install-qt-action. For Qt 6.7+ the aqt arch flag is +# `linux_gcc_64`; the install directory it produces is still `gcc_64`. +ARG QT_VERSION=6.7.0 +RUN aqt install-qt linux desktop "${QT_VERSION}" linux_gcc_64 \ + -O /opt/Qt --modules qtnetworkauth + +ENV Qt6_DIR=/opt/Qt/${QT_VERSION}/gcc_64 \ + CMAKE_PREFIX_PATH=/opt/Qt/${QT_VERSION}/gcc_64 \ + PATH=/opt/Qt/${QT_VERSION}/gcc_64/bin:${PATH} + +# buildcache binary release from the canonical GitLab repo (the GitHub mirror +# is stale at v0.28.4; GitLab progressed to v0.33.0 with the GCC 14 +# transitive-include fixes). Ubuntu 24.04 ships glibc 2.39 so the precompiled +# linux-amd64 binary loads cleanly -- no need for the build-from-source detour +# the RHEL8 image takes. On PATH, so CMakeLists.txt auto-wires it as +# CMAKE_CXX_COMPILER_LAUNCHER via find_program(). +ARG BUILDCACHE_VERSION=v0.33.0 +RUN curl -fsSL -o /tmp/buildcache.tar.gz \ + "https://gitlab.com/api/v4/projects/bits-n-bites%2Fbuildcache/packages/generic/releases/${BUILDCACHE_VERSION}/buildcache-linux-amd64.tar.gz" \ + && tar -xzf /tmp/buildcache.tar.gz -C /tmp \ + && install -m 0755 /tmp/buildcache/bin/buildcache /usr/local/bin/buildcache \ + && rm -rf /tmp/buildcache /tmp/buildcache.tar.gz \ + && buildcache --version + +# ----------------------------------------------------------------------------- +# Stage 2: populate the vcpkg binary cache with the gcc-14 toolchain. +# ----------------------------------------------------------------------------- +FROM base AS vcpkg-builder-gcc + +ENV VCPKG_DEFAULT_BINARY_CACHE=/opt/vcpkg-cache-gcc + +COPY . /src + +# Running the ResInsight CMake configure triggers a vcpkg manifest install, +# which compiles every dependency into VCPKG_DEFAULT_BINARY_CACHE. The configure +# step may then fail past the dependency build; that's tolerated because the +# binary cache is already populated by then. After configure, verify the cache +# contains a plausible number of entries -- ResInsight depends on ~30+ packages, +# so a threshold of 20 catches the failure mode where configure dies early +# (e.g. after building only a handful of leaf deps). +RUN git config --global --add safe.directory '*' \ + && mkdir -p "${VCPKG_DEFAULT_BINARY_CACHE}" \ + && ( cd /src/ThirdParty/vcpkg && ./bootstrap-vcpkg.sh ) \ + && export CC=gcc-14 CXX=g++-14 \ + && { cmake -S /src -B /tmp/cfg-gcc -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true \ + -DRESINSIGHT_ENABLE_UNITY_BUILD=false \ + -DRESINSIGHT_ENABLE_HDF5=false \ + -DRESINSIGHT_ENABLE_GRPC=true \ + -DRESINSIGHT_GRPC_DOWNLOAD_PYTHON_MODULE=true \ + -DCMAKE_TOOLCHAIN_FILE=ThirdParty/vcpkg/scripts/buildsystems/vcpkg.cmake \ + || true; } \ + && vcpkg_zip_count=$(find "${VCPKG_DEFAULT_BINARY_CACHE}" -type f -name '*.zip' | wc -l) \ + && echo "vcpkg gcc binary cache entries: ${vcpkg_zip_count}" \ + && test "${vcpkg_zip_count}" -ge 20 \ + && rm -rf /tmp/cfg-gcc /src + +# ----------------------------------------------------------------------------- +# Stage 3: populate the vcpkg binary cache with the clang-19 toolchain. +# Independent stage so BuildKit can run it in parallel with Stage 2. +# ----------------------------------------------------------------------------- +FROM base AS vcpkg-builder-clang + +ENV VCPKG_DEFAULT_BINARY_CACHE=/opt/vcpkg-cache-clang + +COPY . /src + +RUN git config --global --add safe.directory '*' \ + && mkdir -p "${VCPKG_DEFAULT_BINARY_CACHE}" \ + && ( cd /src/ThirdParty/vcpkg && ./bootstrap-vcpkg.sh ) \ + && export CC=clang-19 CXX=clang++-19 \ + && { cmake -S /src -B /tmp/cfg-clang -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true \ + -DRESINSIGHT_ENABLE_UNITY_BUILD=false \ + -DRESINSIGHT_ENABLE_HDF5=false \ + -DRESINSIGHT_ENABLE_GRPC=true \ + -DRESINSIGHT_GRPC_DOWNLOAD_PYTHON_MODULE=true \ + -DCMAKE_TOOLCHAIN_FILE=ThirdParty/vcpkg/scripts/buildsystems/vcpkg.cmake \ + || true; } \ + && vcpkg_zip_count=$(find "${VCPKG_DEFAULT_BINARY_CACHE}" -type f -name '*.zip' | wc -l) \ + && echo "vcpkg clang binary cache entries: ${vcpkg_zip_count}" \ + && test "${vcpkg_zip_count}" -ge 20 \ + && rm -rf /tmp/cfg-clang /src + +# ----------------------------------------------------------------------------- +# Stage 4: warm the ResInsight buildcache for gcc-14. +# +# Full compile at /src -> /src/cmakebuild. Consumer workflows clone into the +# same /src so buildcache hashes (which include the absolute source path) +# line up. +# ----------------------------------------------------------------------------- +FROM base AS build-warmup-gcc + +# Cap at 4 GB. CI runs read AND write back to /opt/buildcache-gcc, so the cap +# must comfortably exceed (baked entries + per-run accumulation) or LRU +# eviction starts dropping baked entries between nightly rebuilds. +ENV BUILDCACHE_ACCURACY=SLOPPY \ + BUILDCACHE_MAX_CACHE_SIZE=4294967296 + +COPY --from=vcpkg-builder-gcc /opt/vcpkg-cache-gcc /opt/vcpkg-cache-gcc +COPY . /src + +# Unity build is OFF here and in the consumer workflow: with unity ON, a +# single .cpp change invalidates the whole unity blob's cache entry, defeating +# the point. The flag must match between this stage and CI or hit rate +# collapses (the command line is part of the buildcache hash). +# +# RESINSIGHT_TREAT_WARNINGS_AS_ERRORS matches the cache workflow's setting so +# the warmup catches the same warnings CI would. +RUN --mount=type=cache,id=ri-ubuntu24-buildcache-gcc,target=/var/cache/buildcache,sharing=locked \ + git config --global --add safe.directory '*' \ + && ( cd /src/ThirdParty/vcpkg && ./bootstrap-vcpkg.sh ) \ + && export CC=gcc-14 CXX=g++-14 \ + && export VCPKG_FEATURE_FLAGS=binarycaching \ + && export VCPKG_BINARY_SOURCES="clear;files,/opt/vcpkg-cache-gcc,read" \ + && export BUILDCACHE_DIR=/var/cache/buildcache \ + && cmake -S /src -B /src/cmakebuild -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true \ + -DRESINSIGHT_TREAT_WARNINGS_AS_ERRORS=true \ + -DRESINSIGHT_ENABLE_PRECOMPILED_HEADERS=false \ + -DRESINSIGHT_ENABLE_UNITY_BUILD=false \ + -DRESINSIGHT_ENABLE_HDF5=false \ + -DRESINSIGHT_ENABLE_GRPC=true \ + -DRESINSIGHT_GRPC_PYTHON_EXECUTABLE=$(command -v python3) \ + -DRESINSIGHT_GRPC_DOWNLOAD_PYTHON_MODULE=true \ + -DCMAKE_TOOLCHAIN_FILE=ThirdParty/vcpkg/scripts/buildsystems/vcpkg.cmake \ + && cmake --build /src/cmakebuild -- -j"$(nproc)" \ + && buildcache -s \ + && mkdir -p /opt/buildcache-gcc \ + && cp -a /var/cache/buildcache/. /opt/buildcache-gcc/ \ + && test -n "$(ls -A /opt/buildcache-gcc)" \ + && rm -rf /src + +# ----------------------------------------------------------------------------- +# Stage 5: warm the ResInsight buildcache for clang-19. +# ----------------------------------------------------------------------------- +FROM base AS build-warmup-clang + +ENV BUILDCACHE_ACCURACY=SLOPPY \ + BUILDCACHE_MAX_CACHE_SIZE=4294967296 + +COPY --from=vcpkg-builder-clang /opt/vcpkg-cache-clang /opt/vcpkg-cache-clang +COPY . /src + +RUN --mount=type=cache,id=ri-ubuntu24-buildcache-clang,target=/var/cache/buildcache,sharing=locked \ + git config --global --add safe.directory '*' \ + && ( cd /src/ThirdParty/vcpkg && ./bootstrap-vcpkg.sh ) \ + && export CC=clang-19 CXX=clang++-19 \ + && export VCPKG_FEATURE_FLAGS=binarycaching \ + && export VCPKG_BINARY_SOURCES="clear;files,/opt/vcpkg-cache-clang,read" \ + && export BUILDCACHE_DIR=/var/cache/buildcache \ + && cmake -S /src -B /src/cmakebuild -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true \ + -DRESINSIGHT_TREAT_WARNINGS_AS_ERRORS=true \ + -DRESINSIGHT_ENABLE_PRECOMPILED_HEADERS=false \ + -DRESINSIGHT_ENABLE_UNITY_BUILD=false \ + -DRESINSIGHT_ENABLE_HDF5=false \ + -DRESINSIGHT_ENABLE_GRPC=true \ + -DRESINSIGHT_GRPC_PYTHON_EXECUTABLE=$(command -v python3) \ + -DRESINSIGHT_GRPC_DOWNLOAD_PYTHON_MODULE=true \ + -DCMAKE_TOOLCHAIN_FILE=ThirdParty/vcpkg/scripts/buildsystems/vcpkg.cmake \ + && cmake --build /src/cmakebuild -- -j"$(nproc)" \ + && buildcache -s \ + && mkdir -p /opt/buildcache-clang \ + && cp -a /var/cache/buildcache/. /opt/buildcache-clang/ \ + && test -n "$(ls -A /opt/buildcache-clang)" \ + && rm -rf /src + +# ----------------------------------------------------------------------------- +# Stage 6: final image -- base + both vcpkg caches + both warm buildcaches. +# ----------------------------------------------------------------------------- +FROM base + +LABEL org.opencontainers.image.source="https://github.com/OPM/ResInsight" +LABEL org.opencontainers.image.description="Ubuntu 24.04 CI image for ResInsight with gcc-14 + clang-19 toolchains, vcpkg binary caches, and warm ResInsight buildcaches" + +COPY --from=vcpkg-builder-gcc /opt/vcpkg-cache-gcc /opt/vcpkg-cache-gcc +COPY --from=vcpkg-builder-clang /opt/vcpkg-cache-clang /opt/vcpkg-cache-clang +COPY --from=build-warmup-gcc /opt/buildcache-gcc /opt/buildcache-gcc +COPY --from=build-warmup-clang /opt/buildcache-clang /opt/buildcache-clang + +# Default ENV targets the gcc-14 leg (the more common consumer: cache +# workflow's gcc matrix entry, plus the Copilot Coding Agent's setup-steps). +# Consumer workflows that target clang override BUILDCACHE_DIR and +# VCPKG_DEFAULT_BINARY_CACHE at the job level. +ENV BUILDCACHE_DIR=/opt/buildcache-gcc \ + BUILDCACHE_ACCURACY=SLOPPY \ + BUILDCACHE_MAX_CACHE_SIZE=4294967296 \ + VCPKG_DEFAULT_BINARY_CACHE=/opt/vcpkg-cache-gcc diff --git a/.github/docker/README.md b/.github/docker/README.md index f8509d1c5c..ded1b41b7b 100644 --- a/.github/docker/README.md +++ b/.github/docker/README.md @@ -1,13 +1,35 @@ -# RHEL8 CI Docker Image +# ResInsight CI Docker Images -This directory contains the Dockerfile for the prebuilt CI image used by the -RHEL8 unit test workflow. The image bakes in everything that does not change -on a per-PR basis so CI runs only have to recompile the translation units that +This directory contains the Dockerfiles for the prebuilt CI images used by the +Linux CI workflows. Each image bakes in everything that does not change on a +per-PR basis so CI runs only have to recompile the translation units that actually changed. -## What's in the image +| Dockerfile | Image (per repo) | Consumer workflows | +| -------------------- | ----------------------------------------------- | ------------------------------------------------------------- | +| `Dockerfile.rhel8` | `ghcr.io//ci-rhel8` | `rhel8-unit-tests.yml` | +| `Dockerfile.ubuntu24`| `ghcr.io//ci-ubuntu24` | `ResInsightWithCacheLinuxContainer.yml`, `copilot-setup-steps.yml` | -Built from `rockylinux:8` in four stages (see `Dockerfile.rhel8`): +Both images follow the same multi-stage pattern: + +1. A `base` stage with the toolchain(s), Qt, and `buildcache`. +2. One or more `vcpkg-builder` stages that populate per-compiler vcpkg binary + caches. +3. One or more `build-warmup` stages that perform a full ResInsight compile + at `/src -> /src/cmakebuild` and stash the resulting buildcache. +4. A `final` stage that assembles the toolchain plus the baked caches. + +Each image is rebuilt nightly and pushed to GitHub Container Registry. The +image path is computed per repo as `ghcr.io/${GITHUB_REPOSITORY,,}/` so +forks publish to their own namespaces. + +--- + +## RHEL8 image (`Dockerfile.rhel8`) + +### What's in the image + +Built from `rockylinux:8` in four stages: - **Toolchain** — `gcc-toolset-14`, a from-source `libstdc++exp` matched to the toolset's GCC, Qt 6.6.3 (via `aqtinstall`), Python 3.9, Ninja, CMake. @@ -24,11 +46,7 @@ Built from `rockylinux:8` in four stages (see `Dockerfile.rhel8`): clones into the same `/src` path so buildcache hashes line up; mismatched source paths or build flags collapse the hit rate. -## Where the images live - -The image is published to GitHub Container Registry under the repository -namespace, computed by the workflows as -`ghcr.io/${GITHUB_REPOSITORY,,}/ci-rhel8`. In practice: +### Where the image lives | Context | Image path | | -------------------------------------- | --------------------------------------------------- | @@ -37,7 +55,7 @@ namespace, computed by the workflows as Each push tags both `:latest` and a date tag `:YYYY-MM-DD`. -### Viewing the images +### Viewing the image - **Web UI** — package page on GitHub: `https://github.com//ResInsight/pkgs/container/resinsight%2Fci-rhel8` @@ -52,9 +70,9 @@ Each push tags both `:latest` and a date tag `:YYYY-MM-DD`. (Use `/orgs//packages/...` instead of `/users//...` for org-owned packages like the upstream `OPM` namespace.) -## Building the image +### Building the image -### Via GitHub Actions (the supported path) +#### Via GitHub Actions (the supported path) The image is built and pushed by `.github/workflows/build-rhel8-image.yml`: @@ -80,7 +98,7 @@ Cold first build runs the full `ResInsight-tests` compile in the warmup stage (~1.5–2 h). Incremental nightly rebuilds reuse the BuildKit `gha` cache and finish much faster. -### Building locally +#### Building locally Useful for validating Dockerfile changes before pushing to CI. From the repo root: @@ -109,7 +127,7 @@ Notes: docker buildx build ... 2>&1 | tee ri-rhel8-build.log ``` -### Pushing a local image to GHCR +#### Pushing a local image to GHCR Useful when iterating on Dockerfile changes on a fork before merging: push to your own GHCR namespace and trigger `rhel8-unit-tests.yml` on the fork to @@ -136,7 +154,7 @@ verify end-to-end. docker push ghcr.io//resinsight/ci-rhel8:$(date -u +%Y-%m-%d) ``` -## How the image is consumed +### How the image is consumed `.github/workflows/rhel8-unit-tests.yml` runs the unit tests inside this image. It resolves the image name from `${GITHUB_REPOSITORY,,}` so the same @@ -150,7 +168,7 @@ Unity build is disabled in both the warmup and the CI configure for the same reason: with unity on, a single `.cpp` change invalidates the entire unity blob's cache entry. -## Retention +### Retention `.github/workflows/cleanup-rhel8-image.yml` runs daily at 03:00 UTC and deletes old dated tags from GHCR: @@ -170,3 +188,127 @@ gh workflow run cleanup-rhel8-image.yml --repo /ResInsight \ Owner type (User vs Organization) is auto-detected so the workflow runs unchanged on the upstream repo and on forks. + +--- + +## Ubuntu 24.04 image (`Dockerfile.ubuntu24`) + +### What's in the image + +Built from `ubuntu:24.04` in six stages: + +- **Toolchain** — both compilers used by the Linux legs of the cache workflow: + `gcc-14` from Ubuntu's repos and `clang-19` from the upstream LLVM apt repo + (`apt.llvm.org/noble/llvm-toolchain-noble-19`). Plus Python 3.12, Ninja, + CMake, and the GL / Qt apt deps used by the existing non-containerized legs. +- **Qt 6.7.0** at `/opt/Qt/6.7.0/gcc_64`, installed via `aqtinstall` with the + `qtnetworkauth` module. Same version as the cache workflow's Linux legs. +- **buildcache** — `v0.33.0` linux-amd64 binary release from the canonical + GitLab repo (`bits-n-bites/buildcache`). The RHEL8 image builds from source + because Rocky 8's glibc 2.28 is too old for the precompiled binary; Ubuntu + 24.04 ships glibc 2.39 so the binary loads directly. On PATH, so + `CMakeLists.txt` auto-wires it as `CMAKE_CXX_COMPILER_LAUNCHER`. +- **vcpkg binary caches** — two of them, populated by independent stages that + run in parallel under BuildKit: + - `/opt/vcpkg-cache-gcc` (built with `gcc-14`) + - `/opt/vcpkg-cache-clang` (built with `clang-19`) +- **Warm ResInsight buildcaches** — two of them, each populated by a full + ResInsight compile (default target — wider than `ResInsight-tests` alone) + at `/src -> /src/cmakebuild`: + - `/opt/buildcache-gcc` (gcc-14, GRPC on) + - `/opt/buildcache-clang` (clang-19, GRPC on) + +The final stage's ENV targets the gcc-14 leg (`BUILDCACHE_DIR=/opt/buildcache-gcc`, +`VCPKG_DEFAULT_BINARY_CACHE=/opt/vcpkg-cache-gcc`) so the Copilot Coding Agent's +`copilot-setup-steps.yml` works without overrides. The clang consumer overrides +both env vars at the job level. + +### Where the image lives + +| Context | Image path | +| -------------------------------------- | --------------------------------------------------- | +| Upstream (`OPM/ResInsight`) | `ghcr.io/opm/resinsight/ci-ubuntu24` | +| Fork (e.g. `magnesj/ResInsight`) | `ghcr.io/magnesj/resinsight/ci-ubuntu24` | + +Each push tags both `:latest` and a date tag `:YYYY-MM-DD`. + +### Viewing the image + +- **Web UI**: `https://github.com//ResInsight/pkgs/container/resinsight%2Fci-ubuntu24` +- **CLI**: same `gh api` recipe as the RHEL8 image above, with + `ci-ubuntu24` in place of `ci-rhel8`. + +### Building the image + +#### Via GitHub Actions + +`.github/workflows/build-ubuntu24-image.yml`: + +- **Nightly** at 00:00 UTC (the containerized cache workflow runs at 01:00 + UTC). +- **On push** to any of: `Dockerfile.ubuntu24`, `build-ubuntu24-image.yml`, + `vcpkg.json`, `vcpkg-configuration.json`, `.gitmodules`. (Note the default + `vcpkg-configuration.json`, not the RHEL8 variant.) +- **Manually** via `workflow_dispatch`, optionally with `force_rebuild=true` + to skip the "today's image already exists" guard: + + ``` + gh workflow run build-ubuntu24-image.yml --repo /ResInsight --ref \ + -f force_rebuild=true + ``` + +Cold first build runs the full ResInsight compile **twice** (once per +compiler) — ~3–4 h end-to-end. The job timeout is 360 min. Incremental +nightly rebuilds reuse the BuildKit `gha` cache and the per-compiler +`ri-ubuntu24-buildcache-{gcc,clang}` cache mounts, and finish much faster. + +#### Building locally + +``` +docker buildx build \ + --progress=plain \ + --file .github/docker/Dockerfile.ubuntu24 \ + --tag resinsight-ci-ubuntu24:local \ + . +``` + +Notes: + +- BuildKit required (same as the RHEL8 image). +- Cold local build is ~3–4 h on a typical workstation; the resulting image is + ~6–7 GB (two toolchains, two warm caches). +- BuildKit runs the gcc and clang `vcpkg-builder` and `build-warmup` stages in + parallel where it can, so wall-clock time is less than 2× the per-compiler + cost. + +#### Pushing a local image to GHCR + +Same procedure as the RHEL8 image above, with `ci-ubuntu24` in place of +`ci-rhel8`. + +### How the image is consumed + +Two workflows use the image: + +- **`.github/workflows/ResInsightWithCacheLinuxContainer.yml`** — parallel + companion to `ResInsightWithCache.yml`'s Linux legs. Runs both compilers in + a matrix, each leg overriding `BUILDCACHE_DIR` and `VCPKG_BINARY_SOURCES` + to its compiler's baked cache. Source is cloned manually into `/src` + (matching the warmup path) and `defaults.run.working-directory` is set to + `/src` so every step runs there. +- **`.github/workflows/copilot-setup-steps.yml`** — the GitHub Copilot Coding + Agent's pre-flight setup. Uses the image's gcc-14 default ENV; the agent's + first incremental build only recompiles what it edits because the warm + buildcache is already populated. + +In both cases, unity build is OFF (matches the warmup) and the absolute source +path is `/src` (matches the warmup) — drift on either collapses the cache hit +rate. + +### Retention + +`.github/workflows/cleanup-ubuntu24-image.yml` runs daily at 03:00 UTC with +the same rules as the RHEL8 cleanup: keep `:latest`, keep the 3 most recent +versions as a safety floor, delete anything older than 5 days. The +`workflow_dispatch` `dry_run` and `retention_days` inputs are exposed the +same way. diff --git a/.github/workflows/ResInsightWithCacheLinuxContainer.yml b/.github/workflows/ResInsightWithCacheLinuxContainer.yml new file mode 100644 index 0000000000..5a41faf255 --- /dev/null +++ b/.github/workflows/ResInsightWithCacheLinuxContainer.yml @@ -0,0 +1,221 @@ +name: ResInsight Build With Cache (Linux container) + +# Parallel companion to ResInsightWithCache.yml's Linux legs. Runs both the +# Ubuntu 24.04 / gcc-14 and Ubuntu 24.04 / clang-19 builds inside the prebuilt +# image at ghcr.io//ci-ubuntu24:latest -- baked with both toolchains, +# Qt, a per-compiler vcpkg binary cache, and a warm ResInsight buildcache. +# +# See: +# * .github/docker/Dockerfile.ubuntu24 (what the image contains) +# * .github/workflows/build-ubuntu24-image.yml (nightly image build) +# +# The non-containerized Linux legs in ResInsightWithCache.yml continue to run +# in parallel during validation; once this workflow is stable, those legs will +# be removed in a follow-up PR. + +on: + push: + pull_request: + schedule: + # 01:00 UTC -- one hour after the nightly image build at 00:00 UTC. + - cron: '0 1 * * *' + +concurrency: + group: build-linux-container-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + # Resolves the prebuilt CI image name so the workflow works in forks and + # upstream alike (GHCR requires a lowercase repository path). + resolve-image: + runs-on: ubuntu-latest + outputs: + image: ${{ steps.meta.outputs.image }} + steps: + - name: Compute image name + id: meta + run: echo "image=ghcr.io/${GITHUB_REPOSITORY,,}/ci-ubuntu24:latest" >> "$GITHUB_OUTPUT" + + build: + needs: resolve-image + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + strategy: + fail-fast: false + matrix: + config: + - name: "Ubuntu 24.04 gcc-14 (container)" + cc: gcc-14 + cxx: g++-14 + buildcache_dir: /opt/buildcache-gcc + vcpkg_cache: /opt/vcpkg-cache-gcc + execute_pytests: true + publish_to_pypi: true + - name: "Ubuntu 24.04 clang-19 (container)" + cc: clang-19 + cxx: clang++-19 + buildcache_dir: /opt/buildcache-clang + vcpkg_cache: /opt/vcpkg-cache-clang + execute_pytests: false + publish_to_pypi: false + container: + # Prebuilt image with both toolchains, baked vcpkg binary caches, and + # warm ResInsight buildcaches at /opt/buildcache-{gcc,clang}. + # See .github/workflows/build-ubuntu24-image.yml + image: ${{ needs.resolve-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + # Source must live at /src to match the path the image's warmup compiled + # at; buildcache hashes the absolute source path so any drift collapses + # the hit rate. The clone step overrides this default since /src does not + # exist yet at that point. + defaults: + run: + working-directory: /src + env: + BUILD_TYPE: Release + BUILDCACHE_DIR: ${{ matrix.config.buildcache_dir }} + BUILDCACHE_ACCURACY: SLOPPY + + steps: + - name: Clone source into /src + working-directory: / + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # actions/checkout writes inside GITHUB_WORKSPACE (e.g. + # /__w/ResInsight/ResInsight) which would not match the image's baked + # cache path. Clone manually into /src instead. GITHUB_SHA is fetched + # directly (GitHub allows any-SHA fetches on the workflow's own repo). + # + # The insteadOf rewrite is unset after the clone so the token is not + # left in /etc/gitconfig where a later `git config --list` for + # diagnostics could echo it into the log. + run: | + git config --global --add safe.directory '*' + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + git clone --no-checkout "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" /src + cd /src + git fetch --no-tags --depth 1 origin "${GITHUB_SHA}" + git checkout FETCH_HEAD + git submodule update --init --recursive --depth 1 + git config --global --unset url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf + + # /src was just freshly cloned, so the ThirdParty/vcpkg/vcpkg binary + # produced by bootstrap-vcpkg.sh is not present. The image's baked + # vcpkg cache supplies prebuilt dependencies; the binary itself still + # has to exist locally for `vcpkg install` to run. + - name: Bootstrap vcpkg + run: | + cd ThirdParty/vcpkg + ./bootstrap-vcpkg.sh + + - name: Install Python dev dependencies + # rips package is required for pytest and mypy. PEP 668 marks Ubuntu + # 24.04's system Python as externally managed; --break-system-packages + # is the pragmatic path for a single-purpose CI container. + run: | + # rips/_version.py is normally generated by CMake; create a dev + # fallback so the editable install below can resolve the version. + echo '__version__ = "0.0.0.dev"' > GrpcInterface/Python/rips/_version.py + python3 -m pip install --break-system-packages -e GrpcInterface/Python[dev] + + - name: Configure + env: + CC: ${{ matrix.config.cc }} + CXX: ${{ matrix.config.cxx }} + VCPKG_FEATURE_FLAGS: binarycaching + # Read vcpkg dependencies from the per-compiler binary cache baked + # into the image. buildcache is auto-discovered on PATH by + # CMakeLists.txt's find_program() and wired as + # CMAKE_CXX_COMPILER_LAUNCHER -- no explicit setup step is needed. + # Unity build must be OFF to match the warmup build (mismatched + # flag would collapse the cache hit rate). + VCPKG_BINARY_SOURCES: "clear;files,${{ matrix.config.vcpkg_cache }},read" + run: | + cmake -S /src -B /src/cmakebuild -G Ninja \ + -DVCPKG_BUILD_TYPE=release \ + -DCMAKE_INSTALL_PREFIX=/src/cmakebuild/install \ + -DCMAKE_BUILD_TYPE=Release \ + -DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true \ + -DRESINSIGHT_TREAT_WARNINGS_AS_ERRORS=true \ + -DRESINSIGHT_ENABLE_PRECOMPILED_HEADERS=false \ + -DRESINSIGHT_ENABLE_UNITY_BUILD=false \ + -DRESINSIGHT_ENABLE_GRPC=true \ + -DRESINSIGHT_GRPC_PYTHON_EXECUTABLE=$(command -v python3) \ + -DRESINSIGHT_GRPC_DOWNLOAD_PYTHON_MODULE=true \ + -DRESINSIGHT_ENABLE_HDF5=false \ + -DCMAKE_TOOLCHAIN_FILE=ThirdParty/vcpkg/scripts/buildsystems/vcpkg.cmake + + - name: Build + run: cmake --build /src/cmakebuild --target install + + - name: Stats for buildcache + if: always() + run: buildcache -s + + - name: Run Unit Tests + run: /src/cmakebuild/ResInsight-tests + + - name: (Python) Build Python module + run: | + python3 -m pip install --break-system-packages build wheel + cd GrpcInterface/Python + python3 -m build + + - name: (Python) Check types using mypy + run: | + python3 -m pip install --break-system-packages mypy types-protobuf + cd GrpcInterface/Python + python3 -m mypy rips/*.py rips/generated/generated_classes.py + + - name: Run pytest + if: matrix.config.execute_pytests + env: + RESINSIGHT_EXECUTABLE: /src/cmakebuild/ResInsight + run: | + cd GrpcInterface/Python/rips + python3 -m pytest --console + + - name: Upload python distribution folder + if: matrix.config.publish_to_pypi + uses: actions/upload-artifact@v7 + with: + name: python-distribution + path: /src/GrpcInterface/Python/dist + retention-days: 5 + + - name: Upload artifacts + uses: actions/upload-artifact@v7 + with: + name: ResInsight-${{ matrix.config.name }} + path: /src/cmakebuild/install + retention-days: 5 + + pypi-publish: + name: Upload release to PyPI + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/rips + permissions: + id-token: write # IMPORTANT: this permission is mandatory for trusted publishing + + # If this is a tagged release + if: startsWith(github.ref, 'refs/tags') + steps: + - name: Download python distribution folder + uses: actions/download-artifact@v8 + with: + name: python-distribution + path: dist + + - name: Display structure of downloaded files + run: ls -R + + - name: Publish Python package to pypi + # https://docs.pypi.org/trusted-publishers/adding-a-publisher/ + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/build-ubuntu24-image.yml b/.github/workflows/build-ubuntu24-image.yml new file mode 100644 index 0000000000..4f0e62abb9 --- /dev/null +++ b/.github/workflows/build-ubuntu24-image.yml @@ -0,0 +1,109 @@ +name: Build Ubuntu 24.04 CI Image + +# Builds the Docker image used by the containerized Linux cache workflow and +# by the Copilot Coding Agent's setup-steps. The image bakes in both Linux +# toolchains (gcc-14 + clang-19), Qt, a per-compiler vcpkg binary cache, and +# a per-compiler warm ResInsight buildcache so CI runs do not have to +# reprovision them every time. + +on: + schedule: + # Nightly at 00:00 UTC, before the containerized cache workflow's + # scheduled run at 01:00 UTC. + - cron: "0 0 * * *" + push: + paths: + - ".github/docker/Dockerfile.ubuntu24" + - ".github/workflows/build-ubuntu24-image.yml" + - "vcpkg.json" + - "vcpkg-configuration.json" + - ".gitmodules" + workflow_dispatch: + inputs: + force_rebuild: + description: "Rebuild and overwrite today's image even if it already exists in GHCR" + type: boolean + default: false + +concurrency: + group: build-ubuntu24-image + cancel-in-progress: false + +jobs: + build-image: + runs-on: ubuntu-latest + # Cold first build runs the full ResInsight compile twice (once per + # compiler) in the build-warmup stages -- ~3-4 h end-to-end. Incremental + # nightly rebuilds reuse the BuildKit cache mount and finish much faster. + timeout-minutes: 360 + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Compute image name and tag + id: meta + run: | + image="ghcr.io/${GITHUB_REPOSITORY,,}/ci-ubuntu24" + date="$(date -u +%Y-%m-%d)" + repo_name_lc="${GITHUB_REPOSITORY##*/}" + repo_name_lc="${repo_name_lc,,}" + web_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pkgs/container/${repo_name_lc}%2Fci-ubuntu24" + { + echo "image=$image" + echo "date=$date" + echo "web_url=$web_url" + } >> "$GITHUB_OUTPUT" + echo "Image refs : $image:latest" + echo " $image:$date" + echo "GHCR page : $web_url" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # The image build is expensive (cold ~3-4 h). If an image tagged with + # today's UTC date is already in GHCR, treat the build as already done + # for this day and skip. To force a same-day rebuild, dispatch the + # workflow with force_rebuild=true (or delete the dated tag from GHCR + # and rerun). + - name: Check if today's image already exists in GHCR + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force_rebuild }}" == "true" ]]; then + echo "force_rebuild=true -- skipping existence check." + echo "exists=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + target="${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date }}" + if docker buildx imagetools inspect "$target" >/dev/null 2>&1; then + echo "Image $target already exists -- skipping build." + echo "See: ${{ steps.meta.outputs.web_url }}" + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "Image $target not found -- proceeding with build." + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build and push image + if: steps.check.outputs.exists != 'true' + uses: docker/build-push-action@v6 + with: + context: . + file: .github/docker/Dockerfile.ubuntu24 + push: true + tags: | + ${{ steps.meta.outputs.image }}:latest + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/cleanup-ubuntu24-image.yml b/.github/workflows/cleanup-ubuntu24-image.yml new file mode 100644 index 0000000000..0c696195e9 --- /dev/null +++ b/.github/workflows/cleanup-ubuntu24-image.yml @@ -0,0 +1,131 @@ +name: Cleanup Ubuntu 24.04 CI Image Versions + +# Daily cleanup of old dated tags pushed by build-ubuntu24-image.yml. +# +# Retention rules: +# * Versions tagged "latest" are always kept (the consumer workflows pull +# `:latest`, so deleting it would break CI). +# * The most recent MIN_KEEP versions by updated_at are kept regardless of +# age. This is a safety floor: if the last few nightly builds break, we +# can still roll back to a working older image. +# * Of what remains, anything older than RETENTION_DAYS is deleted. +# +# Owner type (User vs Organization) is auto-detected so the same workflow +# runs unchanged on the upstream (OPM/ResInsight, org) and on forks +# (e.g. magnesj/ResInsight, user). + +on: + schedule: + # 03:00 UTC -- after the nightly image build (00:00) and the cache + # workflow's containerized run (01:00) have both finished, so we never + # race them. + - cron: "0 3 * * *" + workflow_dispatch: + inputs: + dry_run: + description: "List versions that would be deleted without deleting" + type: boolean + default: false + retention_days: + description: "Delete versions older than this many days" + type: string + default: "5" + +concurrency: + group: cleanup-ubuntu24-image + cancel-in-progress: false + +jobs: + cleanup: + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Delete old image versions + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ inputs.dry_run == true && 'true' || 'false' }} + RETENTION_DAYS: ${{ inputs.retention_days || '5' }} + MIN_KEEP: "3" + run: | + set -euo pipefail + + owner="${GITHUB_REPOSITORY_OWNER}" + repo_name_lc="${GITHUB_REPOSITORY##*/}" + repo_name_lc="${repo_name_lc,,}" + pkg="${repo_name_lc}/ci-ubuntu24" + pkg_enc="${pkg/\//%2F}" + + # Owner type drives both the list endpoint and the delete endpoint. + # User-owned packages delete via /user/... (authenticated user), org + # packages via /orgs//... + owner_type=$(gh api "/users/$owner" --jq '.type') + case "$owner_type" in + Organization) + list_path="/orgs/$owner/packages/container/$pkg_enc/versions" + del_prefix="/orgs/$owner/packages/container/$pkg_enc/versions" + ;; + User) + list_path="/users/$owner/packages/container/$pkg_enc/versions" + del_prefix="/user/packages/container/$pkg_enc/versions" + ;; + *) + echo "Unknown owner type: $owner_type" >&2 + exit 1 + ;; + esac + + echo "Owner : $owner ($owner_type)" + echo "Package : $pkg" + echo "Retention days : $RETENTION_DAYS" + echo "Safety floor : keep $MIN_KEEP most recent versions" + echo "Dry run : $DRY_RUN" + + # Tolerate 404: the package legitimately does not exist on a fork + # that has never published an image yet. + if ! versions=$(gh api --paginate "$list_path" 2>/tmp/api.err); then + if grep -qiE 'not found|HTTP 404' /tmp/api.err; then + echo "Package $pkg not found on $owner -- nothing to clean up." + exit 0 + fi + cat /tmp/api.err >&2 + exit 1 + fi + + cutoff_epoch=$(date -u -d "$RETENTION_DAYS days ago" +%s) + echo "Cutoff (UTC) : $(date -u -d @"$cutoff_epoch" -Iseconds)" + + # jq pipeline: + # 1. drop versions whose tags include "latest" + # 2. sort newest-first + # 3. skip the first MIN_KEEP (safety floor) + # 4. keep only those older than the cutoff + to_delete=$(jq -c \ + --argjson keep "$MIN_KEEP" \ + --argjson cutoff "$cutoff_epoch" ' + [ .[] | select((.metadata.container.tags // []) | index("latest") | not) ] + | sort_by(.updated_at) | reverse + | .[$keep:] + | [ .[] | select((.updated_at | fromdateiso8601) < $cutoff) ] + ' <<<"$versions") + + count=$(jq 'length' <<<"$to_delete") + echo "Versions to delete: $count" + jq -r '.[] | " - id=\(.id) updated=\(.updated_at) tags=\((.metadata.container.tags // []) | join(","))"' <<<"$to_delete" + + if [[ "$count" -eq 0 ]]; then + echo "Nothing to do." + exit 0 + fi + + if [[ "$DRY_RUN" == "true" ]]; then + echo "Dry run -- not deleting." + exit 0 + fi + + jq -r '.[].id' <<<"$to_delete" | while read -r id; do + echo "DELETE $del_prefix/$id" + gh api -X DELETE "$del_prefix/$id" + done + + echo "Cleanup complete."