diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000..f40b94f7b7 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,36 @@ +# Keep the Docker build context small and avoid shadowing the in-image build +# tree with stale host outputs. The repo's own .gitignore does not apply to +# Docker contexts -- this file is the canonical list of paths excluded from +# `COPY . /src` (used by .github/docker/Dockerfile.rhel8). + +# Local CMake build outputs (host-side; can be tens of GB). +build/ +build_*/ +cmakebuild/ +cmake-build-*/ +out/ + +# vcpkg working directories (only present if vcpkg was run on the host; the +# in-image build re-creates them from the manifest and the baked binary +# cache). +ThirdParty/vcpkg/buildtrees/ +ThirdParty/vcpkg/downloads/ +ThirdParty/vcpkg/packages/ +ThirdParty/vcpkg/installed/ + +# Build/install logs at the repo root. +*.log + +# IDE and editor state. +.vs/ +.vscode/ +.idea/ +*.swp + +# Python caches (rips, GrpcInterface tooling, etc.). +__pycache__/ +*.pyc + +# OS metadata. +.DS_Store +Thumbs.db diff --git a/.github/docker/Dockerfile.rhel8 b/.github/docker/Dockerfile.rhel8 new file mode 100644 index 0000000000..dc5e506ef4 --- /dev/null +++ b/.github/docker/Dockerfile.rhel8 @@ -0,0 +1,209 @@ +# syntax=docker/dockerfile:1 + +# ============================================================================= +# RHEL8 (Rocky Linux 8) CI image for ResInsight. +# +# Bakes in: +# * The toolchain (gcc-toolset-14, libstdc++exp, Qt) +# * A prebuilt vcpkg binary cache (populated from the manifest) +# * A warm ResInsight buildcache populated by a full ResInsight-tests compile +# at the fixed path /src -> /src/cmakebuild. +# +# The CI workflow runs inside this image, checks out into /src (path must +# match the warmup so buildcache hashes line up), and only recompiles +# translation units that actually changed. +# +# Rebuilt nightly by .github/workflows/build-rhel8-image.yml. +# ============================================================================= + +# ----------------------------------------------------------------------------- +# Stage 1: base toolchain +# ----------------------------------------------------------------------------- +FROM rockylinux:8 AS base + +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + +# Base development tools and libraries. +RUN dnf install -y epel-release \ + && dnf config-manager --set-enabled powertools \ + && dnf install -y \ + gcc gcc-c++ make cmake ninja-build git curl zip unzip tar \ + pkgconfig perl which xz flex bison \ + python39 python39-devel python39-pip \ + mesa-libGL-devel mesa-libGLU-devel mesa-libEGL-devel \ + libxkbcommon-devel libxkbcommon-x11-devel \ + xcb-util-keysyms-devel xcb-util-image-devel xcb-util-wm-devel \ + xcb-util-renderutil-devel \ + fontconfig-devel freetype-devel \ + libX11-devel libXext-devel libXrender-devel \ + && dnf install -y \ + gcc-toolset-14 gcc-toolset-14-gcc-c++ \ + gcc-toolset-14-libatomic-devel gcc-toolset-14-libstdc++-devel \ + && dnf clean all + +# Build libstdc++exp from GCC source to match gcc-toolset-14. +RUN source /opt/rh/gcc-toolset-14/enable \ + && GCC_FULL_VERSION=$(gcc -dumpfullversion) \ + && GCC_MAJOR_MINOR=$(echo "$GCC_FULL_VERSION" | cut -d. -f1,2) \ + && GCC_VERSION="${GCC_MAJOR_MINOR}.0" \ + && cd /tmp \ + && curl -LO "https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.xz" \ + && tar xf "gcc-${GCC_VERSION}.tar.xz" \ + && mkdir -p "gcc-${GCC_VERSION}/build-libstdcxx" \ + && cd "gcc-${GCC_VERSION}/build-libstdcxx" \ + && ../libstdc++-v3/configure \ + --prefix=/opt/libstdcxx-exp \ + --disable-multilib \ + --disable-libstdcxx-pch \ + --with-gxx-include-dir="/opt/rh/gcc-toolset-14/root/usr/include/c++/${GCC_VERSION}" \ + && { make -k -j"$(nproc)" || true; } \ + && test -f src/experimental/.libs/libstdc++exp.a \ + && mkdir -p /opt/libstdcxx-exp/lib \ + && cp src/experimental/.libs/libstdc++exp.a /opt/libstdcxx-exp/lib/ \ + && rm -rf /tmp/gcc-* + +# Python and Qt. +ARG QT_VERSION=6.6.3 +RUN alternatives --set python3 /usr/bin/python3.9 \ + && python3 -m pip install --upgrade pip \ + && python3 -m pip install grpcio-tools aqtinstall \ + && python3 -m aqt install-qt linux desktop "${QT_VERSION}" gcc_64 \ + -O /opt/Qt --modules qtnetworkauth + +ENV Qt6_DIR=/opt/Qt/${QT_VERSION}/gcc_64 \ + CMAKE_PREFIX_PATH=/opt/Qt/${QT_VERSION}/gcc_64 \ + PATH=/opt/Qt/${QT_VERSION}/gcc_64/bin:${PATH} + +# buildcache (pinned, built from source). When this binary is on PATH at CMake +# configure time, the project's top-level CMakeLists.txt auto-detects it via +# find_program() and wires CMAKE_CXX_COMPILER_LAUNCHER. No further CMake +# plumbing is needed. +# +# Built from source rather than downloaded because the upstream +# `buildcache-linux.tar.gz` release binary is linked against glibc 2.29+, and +# Rocky Linux 8 ships glibc 2.28 -- the binary loads but crashes the first +# time the dynamic linker touches libm. Compiling here links against the +# image's own glibc and is portable (~1-2 minutes). +# Source from GitLab -- the canonical repo. The GitHub mirror is stale (last +# tag v0.28.4, 2024-03), while GitLab has progressed to v0.33.0 with the GCC +# 14 transitive-include fixes that v0.28.x lacks. +ARG BUILDCACHE_VERSION=v0.33.0 +RUN source /opt/rh/gcc-toolset-14/enable \ + && git clone --depth 1 --branch "${BUILDCACHE_VERSION}" \ + https://gitlab.com/bits-n-bites/buildcache.git /tmp/buildcache-src \ + && cmake -S /tmp/buildcache-src/src -B /tmp/buildcache-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \ + && cmake --build /tmp/buildcache-build --target install \ + && rm -rf /tmp/buildcache-src /tmp/buildcache-build \ + && buildcache --version + +# ----------------------------------------------------------------------------- +# Stage 2: populate the vcpkg binary cache +# ----------------------------------------------------------------------------- +FROM base AS vcpkg-builder + +ENV VCPKG_DEFAULT_BINARY_CACHE=/opt/vcpkg-cache + +COPY . /src + +# Running the ResInsight CMake configure triggers a vcpkg manifest install, +# which compiles every dependency into VCPKG_DEFAULT_BINARY_CACHE. The same +# toolchain is used here and in the final image, so the vcpkg ABI hashes match +# and the cache is reused by CI builds. Later configure steps may fail inside +# the build container; that is tolerated as the dependencies are already built +# by then. +# +# After configure, verify the cache contains a plausible number of entries. +# vcpkg's `files` binary provider stores one .zip per package under +# /<2-hex>/.zip. ResInsight depends on ~30+ packages, so a +# threshold of 20 catches the failure mode where the configure dies early +# (e.g. after building only a handful of leaf deps) while staying robust to +# small manifest churn. +RUN git config --global --add safe.directory '*' \ + && cp /src/vcpkg-configuration-rhel8.json /src/vcpkg-configuration.json \ + && mkdir -p "${VCPKG_DEFAULT_BINARY_CACHE}" \ + && source /opt/rh/gcc-toolset-14/enable \ + && ( cd /src/ThirdParty/vcpkg && ./bootstrap-vcpkg.sh ) \ + && export CC=/opt/rh/gcc-toolset-14/root/usr/bin/gcc \ + && export CXX=/opt/rh/gcc-toolset-14/root/usr/bin/g++ \ + && { cmake -S /src -B /tmp/cfg -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true \ + -DRESINSIGHT_ENABLE_UNITY_BUILD=true \ + -DRESINSIGHT_ENABLE_HDF5=false \ + -DRESINSIGHT_ENABLE_GRPC=false \ + -DRESINSIGHT_GCC_STDCPP_EXP_PATH=/opt/libstdcxx-exp/lib \ + -DCMAKE_TOOLCHAIN_FILE=ThirdParty/vcpkg/scripts/buildsystems/vcpkg.cmake \ + || true; } \ + && vcpkg_zip_count=$(find "${VCPKG_DEFAULT_BINARY_CACHE}" -type f -name '*.zip' | wc -l) \ + && echo "vcpkg binary cache entries: ${vcpkg_zip_count}" \ + && test "${vcpkg_zip_count}" -ge 20 \ + && rm -rf /tmp/cfg /src + +# ----------------------------------------------------------------------------- +# Stage 3: warm the ResInsight buildcache by performing a full compile at +# /src -> /src/cmakebuild. CI runs at the same paths to get cache +# hits (buildcache hashes the absolute source path). +# ----------------------------------------------------------------------------- +FROM base AS build-warmup + +# Cap at 4 GB. The baked cache from this stage is what CI reads, and CI also +# writes back to the same /opt/buildcache, so the cap must be larger than +# (baked entries + per-run accumulation) or LRU eviction starts dropping baked +# entries between nightly rebuilds. 4 GB comfortably exceeds the observed +# warm cache size for a full ResInsight-tests compile. +ENV BUILDCACHE_ACCURACY=SLOPPY \ + BUILDCACHE_MAX_CACHE_SIZE=4294967296 + +COPY --from=vcpkg-builder /opt/vcpkg-cache /opt/vcpkg-cache +COPY . /src + +# The buildcache directory is mounted as a BuildKit cache mount so it persists +# across nightly image rebuilds (each rebuild only recompiles drift since the +# previous run). After the compile finishes the populated cache is copied into +# /opt/buildcache, a real path that is preserved in the image layer. +# +# Unity build is OFF here AND in the consumer workflow: with unity ON, a single +# .cpp change invalidates the whole unity blob's cache entry, defeating the +# point. The flag must match between this stage and CI or hit rate collapses +# (the command line is part of the hash). +RUN --mount=type=cache,id=ri-rhel8-buildcache,target=/var/cache/buildcache,sharing=locked \ + git config --global --add safe.directory '*' \ + && cp /src/vcpkg-configuration-rhel8.json /src/vcpkg-configuration.json \ + && source /opt/rh/gcc-toolset-14/enable \ + && ( cd /src/ThirdParty/vcpkg && ./bootstrap-vcpkg.sh ) \ + && export CC=/opt/rh/gcc-toolset-14/root/usr/bin/gcc \ + && export CXX=/opt/rh/gcc-toolset-14/root/usr/bin/g++ \ + && export VCPKG_FEATURE_FLAGS=binarycaching \ + && export VCPKG_BINARY_SOURCES="clear;files,/opt/vcpkg-cache,read" \ + && export BUILDCACHE_DIR=/var/cache/buildcache \ + && cmake -S /src -B /src/cmakebuild -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true \ + -DRESINSIGHT_ENABLE_UNITY_BUILD=false \ + -DRESINSIGHT_ENABLE_HDF5=false \ + -DRESINSIGHT_ENABLE_GRPC=false \ + -DRESINSIGHT_GCC_STDCPP_EXP_PATH=/opt/libstdcxx-exp/lib \ + -DCMAKE_TOOLCHAIN_FILE=ThirdParty/vcpkg/scripts/buildsystems/vcpkg.cmake \ + && cmake --build /src/cmakebuild --target ResInsight-tests -- -j"$(nproc)" \ + && buildcache -s \ + && mkdir -p /opt/buildcache \ + && cp -a /var/cache/buildcache/. /opt/buildcache/ \ + && test -n "$(ls -A /opt/buildcache)" \ + && rm -rf /src + +# ----------------------------------------------------------------------------- +# Stage 4: final image +# ----------------------------------------------------------------------------- +FROM base + +LABEL org.opencontainers.image.source="https://github.com/OPM/ResInsight" +LABEL org.opencontainers.image.description="RHEL8 CI image for ResInsight with prebuilt toolchain, vcpkg binary cache, and warm ResInsight buildcache" + +COPY --from=vcpkg-builder /opt/vcpkg-cache /opt/vcpkg-cache +COPY --from=build-warmup /opt/buildcache /opt/buildcache + +ENV VCPKG_DEFAULT_BINARY_CACHE=/opt/vcpkg-cache \ + BUILDCACHE_DIR=/opt/buildcache \ + BUILDCACHE_ACCURACY=SLOPPY \ + BUILDCACHE_MAX_CACHE_SIZE=4294967296 diff --git a/.github/docker/README.md b/.github/docker/README.md new file mode 100644 index 0000000000..f8509d1c5c --- /dev/null +++ b/.github/docker/README.md @@ -0,0 +1,172 @@ +# RHEL8 CI Docker Image + +This directory contains the Dockerfile for the prebuilt CI image used by the +RHEL8 unit test workflow. The image bakes in everything that does not change +on a per-PR basis so CI runs only have to recompile the translation units that +actually changed. + +## What's in the image + +Built from `rockylinux:8` in four stages (see `Dockerfile.rhel8`): + +- **Toolchain** — `gcc-toolset-14`, a from-source `libstdc++exp` matched to the + toolset's GCC, Qt 6.6.3 (via `aqtinstall`), Python 3.9, Ninja, CMake. +- **buildcache** — built from source against the image's own glibc (the + upstream `buildcache-linux.tar.gz` binary is linked against glibc 2.29+ and + crashes on Rocky 8's glibc 2.28). Sourced from the canonical GitLab repo + `bits-n-bites/buildcache`. On PATH, so `CMakeLists.txt` auto-wires it as + `CMAKE_CXX_COMPILER_LAUNCHER`. +- **vcpkg binary cache** at `/opt/vcpkg-cache` — populated by running a CMake + configure against the ResInsight manifest, then frozen. Consumed read-only + in CI. +- **Warm ResInsight buildcache** at `/opt/buildcache` — populated by a full + `ResInsight-tests` compile at `/src -> /src/cmakebuild`. The CI workflow + clones into the same `/src` path so buildcache hashes line up; mismatched + source paths or build flags collapse the hit rate. + +## Where the images live + +The image is published to GitHub Container Registry under the repository +namespace, computed by the workflows as +`ghcr.io/${GITHUB_REPOSITORY,,}/ci-rhel8`. In practice: + +| Context | Image path | +| -------------------------------------- | --------------------------------------------------- | +| Upstream (`OPM/ResInsight`) | `ghcr.io/opm/resinsight/ci-rhel8` | +| Fork (e.g. `magnesj/ResInsight`) | `ghcr.io/magnesj/resinsight/ci-rhel8` | + +Each push tags both `:latest` and a date tag `:YYYY-MM-DD`. + +### Viewing the images + +- **Web UI** — package page on GitHub: + `https://github.com//ResInsight/pkgs/container/resinsight%2Fci-rhel8` +- **CLI** — list versions and tags via the GitHub API: + + ``` + gh api -H 'Accept: application/vnd.github+json' \ + /users//packages/container/resinsight%2Fci-rhel8/versions \ + --jq '.[] | {tags: .metadata.container.tags, updated_at: .updated_at}' + ``` + + (Use `/orgs//packages/...` instead of `/users//...` for org-owned + packages like the upstream `OPM` namespace.) + +## Building the image + +### Via GitHub Actions (the supported path) + +The image is built and pushed by `.github/workflows/build-rhel8-image.yml`: + +- **Nightly** at 00:00 UTC (the unit test workflow runs at 02:00 UTC). +- **On push** to any of: `Dockerfile.rhel8`, `build-rhel8-image.yml`, + `vcpkg.json`, `vcpkg-configuration-rhel8.json`, `.gitmodules`. +- **Manually** via `workflow_dispatch`: + + ``` + gh workflow run build-rhel8-image.yml --repo /ResInsight --ref + ``` + + The workflow short-circuits if an image tagged with today's UTC date is + already in GHCR. To force a same-day rebuild (e.g. after iterating on the + Dockerfile within a single day), pass `force_rebuild=true`: + + ``` + gh workflow run build-rhel8-image.yml --repo /ResInsight --ref \ + -f force_rebuild=true + ``` + +Cold first build runs the full `ResInsight-tests` compile in the warmup stage +(~1.5–2 h). Incremental nightly rebuilds reuse the BuildKit `gha` cache and +finish much faster. + +### Building locally + +Useful for validating Dockerfile changes before pushing to CI. From the repo +root: + +``` +docker buildx build \ + --progress=plain \ + --file .github/docker/Dockerfile.rhel8 \ + --tag resinsight-ci-rhel8:local \ + . +``` + +Notes: + +- Requires Docker with BuildKit (default in Docker Desktop / Docker 23+). + The Dockerfile uses `# syntax=docker/dockerfile:1` and a BuildKit cache + mount for the warmup buildcache. +- The build context is ~860 MB even with `.dockerignore` excluding host build + outputs — most of that is `ThirdParty/vcpkg` and other submodules. +- Cold build is ~1.5–2 h end-to-end on a typical workstation; the resulting + image is ~5.3 GB. +- Tee the output to a log file so a crash of the launching shell does not + lose progress (the build itself keeps running in the Docker daemon): + + ``` + docker buildx build ... 2>&1 | tee ri-rhel8-build.log + ``` + +### Pushing a local image to GHCR + +Useful when iterating on Dockerfile changes on a fork before merging: push to +your own GHCR namespace and trigger `rhel8-unit-tests.yml` on the fork to +verify end-to-end. + +1. Ensure your `gh` token has the `write:packages` scope: + + ``` + gh auth refresh -h github.com -s write:packages + ``` + +2. Tag the local image for your fork's GHCR namespace and the desired tags: + + ``` + docker tag resinsight-ci-rhel8:local ghcr.io//resinsight/ci-rhel8:latest + docker tag resinsight-ci-rhel8:local ghcr.io//resinsight/ci-rhel8:$(date -u +%Y-%m-%d) + ``` + +3. Log Docker in to GHCR using the `gh` token and push: + + ``` + gh auth token | docker login ghcr.io -u --password-stdin + docker push ghcr.io//resinsight/ci-rhel8:latest + docker push ghcr.io//resinsight/ci-rhel8:$(date -u +%Y-%m-%d) + ``` + +## How the image is consumed + +`.github/workflows/rhel8-unit-tests.yml` runs the unit tests inside this +image. It resolves the image name from `${GITHUB_REPOSITORY,,}` so the same +workflow definition works on the upstream repo and on forks without +modification. + +The workflow clones the source manually into `/src` (not into +`GITHUB_WORKSPACE`) because buildcache hashes the absolute source path — the +clone path has to match the warmup compile's `/src` for the cache to hit. +Unity build is disabled in both the warmup and the CI configure for the same +reason: with unity on, a single `.cpp` change invalidates the entire unity +blob's cache entry. + +## Retention + +`.github/workflows/cleanup-rhel8-image.yml` runs daily at 03:00 UTC and +deletes old dated tags from GHCR: + +- The `:latest` tag is always kept (the consumer workflow pulls it). +- The 3 most recent versions are kept regardless of age (safety floor so a + string of broken nightly builds can't strand CI without a working image). +- Of the rest, anything older than 5 days is deleted. + +The retention window and dry-run mode are exposed as `workflow_dispatch` +inputs for ad-hoc invocation: + +``` +gh workflow run cleanup-rhel8-image.yml --repo /ResInsight \ + -f dry_run=true -f retention_days=7 +``` + +Owner type (User vs Organization) is auto-detected so the workflow runs +unchanged on the upstream repo and on forks. diff --git a/.github/workflows/build-rhel8-image.yml b/.github/workflows/build-rhel8-image.yml new file mode 100644 index 0000000000..355e058415 --- /dev/null +++ b/.github/workflows/build-rhel8-image.yml @@ -0,0 +1,106 @@ +name: Build RHEL8 CI Image + +# Builds the Docker image used by the RHEL8 unit test workflow. The image bakes +# in the toolchain (gcc-toolset-14, libstdc++exp, Qt) and a prebuilt vcpkg +# binary cache so CI runs do not have to reprovision them every time. + +on: + schedule: + # Nightly at 00:00 UTC, before the RHEL8 unit tests run at 02:00 UTC. + - cron: "0 0 * * *" + push: + paths: + - ".github/docker/Dockerfile.rhel8" + - ".github/workflows/build-rhel8-image.yml" + - "vcpkg.json" + - "vcpkg-configuration-rhel8.json" + - ".gitmodules" + workflow_dispatch: + inputs: + force_rebuild: + description: "Rebuild and overwrite today's image even if it already exists in GHCR" + type: boolean + default: false + +concurrency: + group: build-rhel8-image + cancel-in-progress: false + +jobs: + build-image: + runs-on: ubuntu-latest + # Cold first build runs the full ResInsight compile in the build-warmup + # stage (~1.5-2 h). Incremental nightly rebuilds reuse the BuildKit cache + # mount and finish much faster. + timeout-minutes: 180 + permissions: + contents: read + packages: write + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: recursive + + - name: Compute image name and tag + id: meta + run: | + image="ghcr.io/${GITHUB_REPOSITORY,,}/ci-rhel8" + date="$(date -u +%Y-%m-%d)" + repo_name_lc="${GITHUB_REPOSITORY##*/}" + repo_name_lc="${repo_name_lc,,}" + web_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/pkgs/container/${repo_name_lc}%2Fci-rhel8" + { + echo "image=$image" + echo "date=$date" + echo "web_url=$web_url" + } >> "$GITHUB_OUTPUT" + echo "Image refs : $image:latest" + echo " $image:$date" + echo "GHCR page : $web_url" + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # The image build is expensive (cold ~1.5-2 h). If an image tagged with + # today's UTC date is already in GHCR, treat the build as already done + # for this day and skip. To force a same-day rebuild, dispatch the + # workflow with force_rebuild=true (or delete the dated tag from GHCR + # and rerun). + - name: Check if today's image already exists in GHCR + id: check + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.force_rebuild }}" == "true" ]]; then + echo "force_rebuild=true -- skipping existence check." + echo "exists=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + target="${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date }}" + if docker buildx imagetools inspect "$target" >/dev/null 2>&1; then + echo "Image $target already exists -- skipping build." + echo "See: ${{ steps.meta.outputs.web_url }}" + echo "exists=true" >> "$GITHUB_OUTPUT" + else + echo "Image $target not found -- proceeding with build." + echo "exists=false" >> "$GITHUB_OUTPUT" + fi + + - name: Build and push image + if: steps.check.outputs.exists != 'true' + uses: docker/build-push-action@v6 + with: + context: . + file: .github/docker/Dockerfile.rhel8 + push: true + tags: | + ${{ steps.meta.outputs.image }}:latest + ${{ steps.meta.outputs.image }}:${{ steps.meta.outputs.date }} + cache-from: type=gha + cache-to: type=gha,mode=max diff --git a/.github/workflows/cleanup-rhel8-image.yml b/.github/workflows/cleanup-rhel8-image.yml new file mode 100644 index 0000000000..e456e55f84 --- /dev/null +++ b/.github/workflows/cleanup-rhel8-image.yml @@ -0,0 +1,130 @@ +name: Cleanup RHEL8 CI Image Versions + +# Daily cleanup of old dated tags pushed by build-rhel8-image.yml. +# +# Retention rules: +# * Versions tagged "latest" are always kept (the consumer workflow pulls +# `:latest`, so deleting it would break CI). +# * The most recent MIN_KEEP versions by updated_at are kept regardless of +# age. This is a safety floor: if the last few nightly builds break, we +# can still roll back to a working older image. +# * Of what remains, anything older than RETENTION_DAYS is deleted. +# +# Owner type (User vs Organization) is auto-detected so the same workflow +# runs unchanged on the upstream (OPM/ResInsight, org) and on forks +# (e.g. magnesj/ResInsight, user). + +on: + schedule: + # 03:00 UTC -- after the nightly image build (00:00) and the unit-test + # consumer (02:00) have both finished, so we never race them. + - cron: "0 3 * * *" + workflow_dispatch: + inputs: + dry_run: + description: "List versions that would be deleted without deleting" + type: boolean + default: false + retention_days: + description: "Delete versions older than this many days" + type: string + default: "5" + +concurrency: + group: cleanup-rhel8-image + cancel-in-progress: false + +jobs: + cleanup: + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - name: Delete old image versions + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + DRY_RUN: ${{ inputs.dry_run == true && 'true' || 'false' }} + RETENTION_DAYS: ${{ inputs.retention_days || '5' }} + MIN_KEEP: "3" + run: | + set -euo pipefail + + owner="${GITHUB_REPOSITORY_OWNER}" + repo_name_lc="${GITHUB_REPOSITORY##*/}" + repo_name_lc="${repo_name_lc,,}" + pkg="${repo_name_lc}/ci-rhel8" + pkg_enc="${pkg/\//%2F}" + + # Owner type drives both the list endpoint and the delete endpoint. + # User-owned packages delete via /user/... (authenticated user), org + # packages via /orgs//... + owner_type=$(gh api "/users/$owner" --jq '.type') + case "$owner_type" in + Organization) + list_path="/orgs/$owner/packages/container/$pkg_enc/versions" + del_prefix="/orgs/$owner/packages/container/$pkg_enc/versions" + ;; + User) + list_path="/users/$owner/packages/container/$pkg_enc/versions" + del_prefix="/user/packages/container/$pkg_enc/versions" + ;; + *) + echo "Unknown owner type: $owner_type" >&2 + exit 1 + ;; + esac + + echo "Owner : $owner ($owner_type)" + echo "Package : $pkg" + echo "Retention days : $RETENTION_DAYS" + echo "Safety floor : keep $MIN_KEEP most recent versions" + echo "Dry run : $DRY_RUN" + + # Tolerate 404: the package legitimately does not exist on a fork + # that has never published an image yet. + if ! versions=$(gh api --paginate "$list_path" 2>/tmp/api.err); then + if grep -qiE 'not found|HTTP 404' /tmp/api.err; then + echo "Package $pkg not found on $owner -- nothing to clean up." + exit 0 + fi + cat /tmp/api.err >&2 + exit 1 + fi + + cutoff_epoch=$(date -u -d "$RETENTION_DAYS days ago" +%s) + echo "Cutoff (UTC) : $(date -u -d @"$cutoff_epoch" -Iseconds)" + + # jq pipeline: + # 1. drop versions whose tags include "latest" + # 2. sort newest-first + # 3. skip the first MIN_KEEP (safety floor) + # 4. keep only those older than the cutoff + to_delete=$(jq -c \ + --argjson keep "$MIN_KEEP" \ + --argjson cutoff "$cutoff_epoch" ' + [ .[] | select((.metadata.container.tags // []) | index("latest") | not) ] + | sort_by(.updated_at) | reverse + | .[$keep:] + | [ .[] | select((.updated_at | fromdateiso8601) < $cutoff) ] + ' <<<"$versions") + + count=$(jq 'length' <<<"$to_delete") + echo "Versions to delete: $count" + jq -r '.[] | " - id=\(.id) updated=\(.updated_at) tags=\((.metadata.container.tags // []) | join(","))"' <<<"$to_delete" + + if [[ "$count" -eq 0 ]]; then + echo "Nothing to do." + exit 0 + fi + + if [[ "$DRY_RUN" == "true" ]]; then + echo "Dry run -- not deleting." + exit 0 + fi + + jq -r '.[].id' <<<"$to_delete" | while read -r id; do + echo "DELETE $del_prefix/$id" + gh api -X DELETE "$del_prefix/$id" + done + + echo "Cleanup complete." diff --git a/.github/workflows/rhel8-unit-tests.yml b/.github/workflows/rhel8-unit-tests.yml index 8b7d49f573..06d95544c5 100644 --- a/.github/workflows/rhel8-unit-tests.yml +++ b/.github/workflows/rhel8-unit-tests.yml @@ -5,145 +5,75 @@ on: schedule: # Nightly at 2am UTC - cron: "0 2 * * *" - + jobs: - rhel8-build-and-test: + # Resolves the prebuilt CI image name so it works in forks and upstream alike + # (GHCR requires a lowercase repository path). + resolve-image: runs-on: ubuntu-latest + outputs: + image: ${{ steps.meta.outputs.image }} + steps: + - name: Compute image name + id: meta + run: echo "image=ghcr.io/${GITHUB_REPOSITORY,,}/ci-rhel8:latest" >> "$GITHUB_OUTPUT" + + rhel8-build-and-test: + needs: resolve-image + runs-on: ubuntu-latest + permissions: + contents: read + packages: read container: - image: rockylinux:8 + # Prebuilt image with toolchain, baked vcpkg binary cache, and a warm + # ResInsight buildcache at /opt/buildcache. + # See .github/workflows/build-rhel8-image.yml + image: ${{ needs.resolve-image.outputs.image }} + credentials: + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Source must live at /src to match the path the image's warmup compiled + # at; buildcache hashes the absolute source path so any drift collapses + # the hit rate. The clone step overrides this default since /src does not + # exist yet at that point. + defaults: + run: + working-directory: /src steps: - - name: Install EPEL and development tools - run: | - dnf install -y epel-release - dnf config-manager --set-enabled powertools - dnf install -y \ - gcc \ - gcc-c++ \ - make \ - cmake \ - ninja-build \ - git \ - curl \ - zip \ - unzip \ - tar \ - pkgconfig \ - perl \ - which \ - python39 \ - python39-devel \ - python39-pip \ - mesa-libGL-devel \ - mesa-libGLU-devel \ - mesa-libEGL-devel \ - libxkbcommon-devel \ - libxkbcommon-x11-devel \ - xcb-util-keysyms-devel \ - xcb-util-image-devel \ - xcb-util-wm-devel \ - xcb-util-renderutil-devel \ - fontconfig-devel \ - freetype-devel \ - libX11-devel \ - libXext-devel \ - libXrender-devel \ - flex \ - bison \ - xz - - - name: Install GCC Toolset 14 - run: | - dnf install -y gcc-toolset-14 gcc-toolset-14-gcc-c++ gcc-toolset-14-libatomic-devel gcc-toolset-14-libstdc++-devel - - - name: Build libstdc++exp - run: | - source /opt/rh/gcc-toolset-14/enable - - # Get full GCC version (e.g., 14.2.1) - GCC_FULL_VERSION=$(gcc -dumpfullversion) - # Get major.minor version for download (e.g., 14.2.0 - releases use .0 patch) - GCC_MAJOR_MINOR=$(echo $GCC_FULL_VERSION | cut -d. -f1,2) - GCC_VERSION="${GCC_MAJOR_MINOR}.0" - echo "GCC full version: $GCC_FULL_VERSION" - echo "GCC download version: $GCC_VERSION" - - # Download GCC source - cd /tmp - curl -LO https://ftp.gnu.org/gnu/gcc/gcc-${GCC_VERSION}/gcc-${GCC_VERSION}.tar.xz - tar xf gcc-${GCC_VERSION}.tar.xz - - # Build libstdc++exp - mkdir -p gcc-${GCC_VERSION}/build-libstdcxx - cd gcc-${GCC_VERSION}/build-libstdcxx - - ../libstdc++-v3/configure \ - --prefix=/opt/libstdcxx-exp \ - --disable-multilib \ - --disable-libstdcxx-pch \ - --with-gxx-include-dir=/opt/rh/gcc-toolset-14/root/usr/include/c++/${GCC_VERSION} - - # Build the experimental library (use -k to continue past unrelated failures like tzdb.cc) - make -k -j$(nproc) || true - - # Verify and install the library - if [ ! -f src/experimental/.libs/libstdc++exp.a ]; then - echo "ERROR: libstdc++exp.a was not built" - exit 1 - fi - mkdir -p /opt/libstdcxx-exp/lib - cp src/experimental/.libs/libstdc++exp.a /opt/libstdcxx-exp/lib/ - - echo "libstdc++exp built successfully" - ls -la /opt/libstdcxx-exp/lib/ - - - name: Configure git safe directory + - name: Clone source into /src + working-directory: / + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # actions/checkout writes inside GITHUB_WORKSPACE (e.g. + # /__w/ResInsight/ResInsight) which would not match the image's baked + # cache path. Clone manually into /src instead. GITHUB_SHA is fetched + # directly (GitHub allows any-SHA fetches on the workflow's own repo). + # + # The insteadOf rewrite is unset after the clone so the token is not + # left in /etc/gitconfig where a later `git config --list` for + # diagnostics could echo it into the log. run: | git config --global --add safe.directory '*' + git config --global url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf "https://github.com/" + git clone --no-checkout "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git" /src + cd /src + git fetch --no-tags --depth 1 origin "${GITHUB_SHA}" + git checkout FETCH_HEAD + git submodule update --init --recursive --depth 1 + git config --global --unset url."https://x-access-token:${GH_TOKEN}@github.com/".insteadOf - - name: Checkout - uses: actions/checkout@v6 - with: - submodules: recursive - - - name: Setup Python - run: | - alternatives --set python3 /usr/bin/python3.9 - python3 --version - python3 -m pip install --upgrade pip - python3 -m pip install grpcio-tools - - - name: Get Python executable path - id: python-path - run: | - echo "PYTHON_EXECUTABLE=$(python3 -c 'import sys; import pathlib; print(pathlib.PurePath(sys.executable).as_posix())')" >> $GITHUB_OUTPUT - - - name: Install Qt 6 using aqtinstall - run: | - python3 -m pip install aqtinstall - python3 -m aqt install-qt linux desktop 6.6.3 gcc_64 -O /opt/Qt --modules qtnetworkauth - echo "Qt6_DIR=/opt/Qt/6.6.3/gcc_64" >> $GITHUB_ENV - echo "CMAKE_PREFIX_PATH=/opt/Qt/6.6.3/gcc_64" >> $GITHUB_ENV - echo "/opt/Qt/6.6.3/gcc_64/bin" >> $GITHUB_PATH - + # /src was just freshly cloned, so the ThirdParty/vcpkg/vcpkg binary + # produced by bootstrap-vcpkg.sh is not present. The image's baked + # /opt/vcpkg-cache supplies prebuilt dependencies; the binary itself + # still has to exist locally for `vcpkg install` to run. - name: Bootstrap vcpkg run: | source /opt/rh/gcc-toolset-14/enable cd ThirdParty/vcpkg ./bootstrap-vcpkg.sh - - name: Get vcpkg submodule SHA - id: vcpkg-sha - shell: bash - run: echo "sha=$(git rev-parse HEAD:ThirdParty/vcpkg)" >> $GITHUB_OUTPUT - - - name: Restore vcpkg cache - id: vcpkg-cache - uses: CeetronSolutions/vcpkg-cache@copilot/optimize-cache-storage-structure - with: - cache-key: ${{ runner.os }}-g++-${{ steps.vcpkg-sha.outputs.sha }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }} - prefix: vcpkg-g++/ - - name: Use RHEL8 vcpkg configuration run: cp vcpkg-configuration-rhel8.json vcpkg-configuration.json @@ -152,13 +82,19 @@ jobs: CC: /opt/rh/gcc-toolset-14/root/usr/bin/gcc CXX: /opt/rh/gcc-toolset-14/root/usr/bin/g++ VCPKG_FEATURE_FLAGS: "binarycaching" - VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},readwrite" + # Read vcpkg dependencies from the binary cache baked into the image. + # buildcache is auto-discovered on PATH by CMakeLists.txt's + # find_program() and wired as CMAKE_CXX_COMPILER_LAUNCHER -- no + # explicit setup step is needed. BUILDCACHE_DIR etc. are baked in + # the image ENV. Unity build must be OFF to match the warmup build + # (mismatched flag would collapse the cache hit rate). + VCPKG_BINARY_SOURCES: "clear;files,/opt/vcpkg-cache,read" run: | source /opt/rh/gcc-toolset-14/enable - cmake -S . -B cmakebuild -G Ninja \ + cmake -S /src -B /src/cmakebuild -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true \ - -DRESINSIGHT_ENABLE_UNITY_BUILD=true \ + -DRESINSIGHT_ENABLE_UNITY_BUILD=false \ -DRESINSIGHT_ENABLE_HDF5=false \ -DRESINSIGHT_ENABLE_GRPC=false \ -DRESINSIGHT_GCC_STDCPP_EXP_PATH=/opt/libstdcxx-exp/lib \ @@ -167,9 +103,13 @@ jobs: - name: Build run: | source /opt/rh/gcc-toolset-14/enable - cmake --build cmakebuild --target ResInsight-tests -- -j$(nproc) + cmake --build /src/cmakebuild --target ResInsight-tests -- -j$(nproc) + + - name: Buildcache stats + if: always() + run: buildcache -s - name: Run Unit Tests run: | source /opt/rh/gcc-toolset-14/enable - ./cmakebuild/ResInsight-tests + /src/cmakebuild/ResInsight-tests