mirror of
https://github.com/OPM/ResInsight.git
synced 2026-11-01 16:43:03 -06:00
Magne Sjaastad
88f471d649
The previous fix added VCToolsVersion to the Windows vcpkg cache key, but it is still too coarse. vcpkg keys each binary's ABI on a SHA-1 of the actual cl.exe binary content (ThirdParty/vcpkg/scripts/detect_compiler/CMakeLists.txt), not on the VCToolsVersion folder name. GitHub ships rebuilt cl.exe binaries under the same VCToolsVersion across runner image releases, so vcpkg's ABI hash changes while VCToolsVersion stays fixed. The restored binaries are then rejected and vcpkg rebuilds everything. Because the cache key did not change, the action skips saving the freshly built binaries, so the stale entry never refreshes and the rebuild loop persists across runs. Observed on two dev runs with identical VCToolsVersion (14.44.35207), SDK and vcpkg SHA but different runner images (win22/20260616.203 vs win22/20260622.215): the first restored 102 packages from the binary cache, the second restored 0 and rebuilt all of them. Hash cl.exe directly so the cache key tracks exactly what vcpkg tracks: a cl.exe change yields a new key, one cold rebuild, then a successful save under the new key, while image bumps that do not touch cl.exe keep restoring from cache.
341 lines
13 KiB
YAML
341 lines
13 KiB
YAML
name: ResInsight Build With Cache
|
|
|
|
on:
|
|
push:
|
|
pull_request:
|
|
schedule:
|
|
# Every day at 1am to make sure we have a build cache for the current date
|
|
# build cache is
|
|
- cron: '0 1 * * *'
|
|
|
|
concurrency:
|
|
group: build-${{ github.event.pull_request.number || github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
build:
|
|
runs-on: ${{ matrix.config.os }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
build_type: [Release]
|
|
config:
|
|
- {
|
|
name: "Windows_2022_qt_610",
|
|
os: windows-2022,
|
|
cc: "cl",
|
|
cxx: "cl",
|
|
build-python-module: true,
|
|
execute-unit-tests: true,
|
|
execute-pytests: true,
|
|
unity-build: true,
|
|
publish-to-pypi: false,
|
|
vcpkg-bootstrap: bootstrap-vcpkg.bat,
|
|
qt-version: 6.10.1,
|
|
ri-unit-test-path: "ResInsight-tests",
|
|
}
|
|
- {
|
|
name: "Ubuntu 24.04 gcc",
|
|
os: ubuntu-24.04,
|
|
cc: "gcc-14",
|
|
cxx: "g++-14",
|
|
build-python-module: true,
|
|
execute-unit-tests: true,
|
|
execute-pytests: true,
|
|
unity-build: false,
|
|
publish-to-pypi: true,
|
|
vcpkg-bootstrap: bootstrap-vcpkg.sh,
|
|
qt-version: 6.7.0,
|
|
ri-unit-test-path: "ResInsight-tests",
|
|
}
|
|
- {
|
|
name: "Ubuntu 24.04 clang-19",
|
|
os: ubuntu-24.04,
|
|
cc: "clang-19",
|
|
cxx: "clang++-19",
|
|
build-python-module: true,
|
|
execute-unit-tests: true,
|
|
execute-pytests: false,
|
|
unity-build: false,
|
|
publish-to-pypi: false,
|
|
vcpkg-bootstrap: bootstrap-vcpkg.sh,
|
|
qt-version: 6.7.0,
|
|
ri-unit-test-path: "ResInsight-tests",
|
|
}
|
|
env:
|
|
BUILD_TYPE: Release
|
|
BUILDCACHE_DIR: ${{ github.workspace }}/buildcache_dir
|
|
BUILDCACHE_ACCURACY: SLOPPY
|
|
VCPKG_BINARY_SOURCES: "clear;x-gha,readwrite"
|
|
# Supply-chain hardening: never install Python packages published within
|
|
# the last week, to avoid pulling freshly published (potentially
|
|
# compromised) releases. Honored by all `uv pip install` invocations below.
|
|
UV_EXCLUDE_NEWER: "1 week"
|
|
|
|
steps:
|
|
- name: Checkout
|
|
uses: actions/checkout@v7
|
|
with:
|
|
submodules: true
|
|
|
|
- name: Set up Python
|
|
uses: actions/setup-python@v6
|
|
with:
|
|
python-version: "3.12"
|
|
- name: Display Python version
|
|
run: python -c "import sys; print(sys.version)"
|
|
|
|
- name: Set up uv
|
|
uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0
|
|
|
|
- name: Get Python executable path
|
|
shell: bash
|
|
id: python-path
|
|
run: echo "PYTHON_EXECUTABLE=$(python -c 'import sys; import pathlib; print (pathlib.PurePath(sys.executable).as_posix())')" >> $GITHUB_OUTPUT
|
|
|
|
- name: Print Python path
|
|
run: echo "${{ steps.python-path.outputs.PYTHON_EXECUTABLE }}"
|
|
|
|
- name: Install dependencies
|
|
run: |
|
|
# rips/_version.py is normally generated by CMake; create a dev
|
|
# fallback so the editable install below can resolve the version.
|
|
echo '__version__ = "0.0.0.dev"' > GrpcInterface/Python/rips/_version.py
|
|
uv pip install --system -e "GrpcInterface/Python[dev]"
|
|
|
|
- name: Use MSVC (Windows)
|
|
if: contains(matrix.config.os, 'windows')
|
|
uses: TheMrMilchmann/setup-msvc-dev@v4
|
|
with:
|
|
arch: x64
|
|
|
|
- name: Setup buildcache
|
|
uses: CeetronSolutions/setup-buildcache-action@v1.1
|
|
|
|
- name: Get current date
|
|
id: current-time
|
|
shell: bash
|
|
run: echo "formattedTime=$(date -u +%Y-%m-%d)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Print time stamp
|
|
run: echo "timestamp ${{ steps.current-time.outputs.formattedTime }}"
|
|
|
|
- name: Cache Buildcache
|
|
id: cache-buildcache
|
|
uses: actions/cache@v5
|
|
with:
|
|
path: ${{ env.BUILDCACHE_DIR }}
|
|
key: ${{ matrix.config.os }}-${{ matrix.config.cc }}-${{ matrix.config.qt-version }}-cache-v03-${{ steps.current-time.outputs.formattedTime }}
|
|
- name: Create Folder for buildcache
|
|
run: New-Item ${{ env.BUILDCACHE_DIR }} -ItemType "directory" -Force
|
|
shell: pwsh
|
|
|
|
- name: Set apt mirror
|
|
# see https://github.com/actions/runner-images/issues/7048
|
|
if: contains( matrix.config.os, 'ubuntu')
|
|
run: |
|
|
# make sure there is a `\t` between URL and `priority:*` attributes
|
|
printf 'http://azure.archive.ubuntu.com/ubuntu priority:1\n' | sudo tee /etc/apt/mirrors.txt
|
|
curl http://mirrors.ubuntu.com/mirrors.txt | sudo tee --append /etc/apt/mirrors.txt
|
|
sudo sed -i 's/http:\/\/azure.archive.ubuntu.com\/ubuntu\//mirror+file:\/etc\/apt\/mirrors.txt/' /etc/apt/sources.list
|
|
|
|
- name: Install Linux dependencies
|
|
if: contains( matrix.config.os, 'ubuntu')
|
|
run: |
|
|
sudo apt-get update --option="APT::Acquire::Retries=3"
|
|
sudo apt-get install --option="APT::Acquire::Retries=3" libxkbcommon-x11-0 libgl1-mesa-dev mesa-common-dev libglfw3-dev libglu1-mesa-dev libhdf5-dev
|
|
|
|
- name: Install clang-19
|
|
if: contains( matrix.config.cc, 'clang')
|
|
run: |
|
|
set -e
|
|
|
|
# Add LLVM repository
|
|
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
|
|
echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-19 main" | sudo tee /etc/apt/sources.list.d/llvm-19.list
|
|
echo "deb-src http://apt.llvm.org/noble/ llvm-toolchain-noble-19 main" | sudo tee -a /etc/apt/sources.list.d/llvm-19.list
|
|
|
|
# Update package list and install clang-19
|
|
sudo apt-get update --option="APT::Acquire::Retries=3"
|
|
sudo apt-get install --option="APT::Acquire::Retries=3" -y \
|
|
clang-19 \
|
|
lldb-19 \
|
|
lld-19 \
|
|
clangd-19 \
|
|
clang-tidy-19 \
|
|
clang-format-19 \
|
|
clang-tools-19 \
|
|
libc++-19-dev \
|
|
libc++abi-19-dev \
|
|
libomp-19-dev
|
|
|
|
# Verify installation
|
|
clang-19 --version
|
|
clang++-19 --version
|
|
echo "clang-19 installation completed successfully"
|
|
|
|
- name: Install gcc-14
|
|
if: contains( matrix.config.cc, 'gcc-14')
|
|
run: |
|
|
sudo apt-get install --option="APT::Acquire::Retries=3" gcc-14 g++-14
|
|
|
|
- name: Install Qt
|
|
uses: CeetronSolutions/install-qt-action@bump-node24
|
|
with:
|
|
version: ${{ matrix.config.qt-version }}
|
|
dir: "${{ github.workspace }}/Qt/"
|
|
cache: true
|
|
setup-python: false
|
|
modules: "qtnetworkauth"
|
|
|
|
- name: vcpkg bootstrap
|
|
run: |
|
|
ThirdParty/vcpkg/${{ matrix.config.vcpkg-bootstrap }}
|
|
|
|
- name: Get vcpkg submodule SHA
|
|
id: vcpkg-sha
|
|
shell: bash
|
|
run: echo "sha=$(git rev-parse HEAD:ThirdParty/vcpkg)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Get MSVC compiler hash (Windows)
|
|
# vcpkg keys each binary's ABI on a SHA-1 of the actual cl.exe binary
|
|
# content (see ThirdParty/vcpkg/scripts/detect_compiler/CMakeLists.txt),
|
|
# not on the VCToolsVersion folder name. GitHub ships rebuilt cl.exe
|
|
# binaries under the same VCToolsVersion across runner image releases, so
|
|
# vcpkg's ABI hash changes while VCToolsVersion stays fixed; the restored
|
|
# binaries are then rejected and vcpkg rebuilds everything. Because the
|
|
# cache key did not change, the action skips saving the freshly built
|
|
# binaries, so the stale entry never refreshes and the rebuild loop
|
|
# persists across runs. Hash cl.exe directly so the cache key tracks
|
|
# exactly what vcpkg tracks: a cl.exe change yields a new key, one cold
|
|
# rebuild, then a successful save under the new key. cl.exe is on PATH
|
|
# via the "Use MSVC (Windows)" step above.
|
|
id: compiler-hash
|
|
if: contains(matrix.config.os, 'windows')
|
|
shell: bash
|
|
run: |
|
|
cl_path=$(cygpath -u "$(where.exe cl.exe | head -n 1 | tr -d '\r')")
|
|
echo "Using compiler: $cl_path"
|
|
echo "hash=$(sha1sum "$cl_path" | cut -d' ' -f1)" >> $GITHUB_OUTPUT
|
|
|
|
- name: Restore vcpkg cache
|
|
id: vcpkg-cache
|
|
uses: CeetronSolutions/vcpkg-cache@copilot/optimize-cache-storage-structure
|
|
with:
|
|
cache-key: ${{ runner.os }}-${{ matrix.config.cxx }}-${{ steps.compiler-hash.outputs.hash }}-${{ steps.vcpkg-sha.outputs.sha }}-${{ hashFiles('vcpkg.json', 'vcpkg-configuration.json') }}
|
|
prefix: vcpkg-${{ matrix.config.cxx}}/
|
|
|
|
- name: Print CMake version
|
|
shell: bash
|
|
run: |
|
|
if [ "$RUNNER_OS" = "Windows" ]; then
|
|
EXEC_PATH=$(where.exe cmake | head -n 1)
|
|
else
|
|
EXEC_PATH=$(which cmake)
|
|
fi
|
|
echo "Executable path: $EXEC_PATH"
|
|
"$EXEC_PATH" --version
|
|
|
|
- name: Configure
|
|
shell: bash
|
|
env:
|
|
VCPKG_FEATURE_FLAGS: "binarycaching" # Possibly redundant, but explicitly sets the binary caching feature flag
|
|
VCPKG_BINARY_SOURCES: "clear;files,${{ steps.vcpkg-cache.outputs.path }},readwrite"
|
|
CC: ${{ matrix.config.cc }}
|
|
CXX: ${{ matrix.config.cxx }}
|
|
run: >
|
|
cmake -S . -B cmakebuild
|
|
-DVCPKG_BUILD_TYPE=release
|
|
-DCMAKE_INSTALL_PREFIX=cmakebuild/install
|
|
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }}
|
|
-DRESINSIGHT_INCLUDE_APPLICATION_UNIT_TESTS=true
|
|
-DRESINSIGHT_TREAT_WARNINGS_AS_ERRORS=true
|
|
-DRESINSIGHT_ENABLE_PRECOMPILED_HEADERS=false
|
|
-DRESINSIGHT_ENABLE_UNITY_BUILD=${{ matrix.config.unity-build }}
|
|
-DRESINSIGHT_ENABLE_GRPC=${{ matrix.config.build-python-module }}
|
|
-DRESINSIGHT_GRPC_PYTHON_EXECUTABLE=${{ steps.python-path.outputs.PYTHON_EXECUTABLE }}
|
|
-DRESINSIGHT_GRPC_DOWNLOAD_PYTHON_MODULE=true
|
|
-DRESINSIGHT_ENABLE_HDF5=false
|
|
-DCMAKE_TOOLCHAIN_FILE=ThirdParty/vcpkg/scripts/buildsystems/vcpkg.cmake
|
|
-G Ninja
|
|
|
|
- name: Build
|
|
run: |
|
|
cmake --build cmakebuild --target install
|
|
|
|
- name: Stats for buildcache
|
|
run: buildcache -s
|
|
|
|
- name: Run Unit Tests
|
|
if: matrix.config.execute-unit-tests
|
|
shell: bash
|
|
run: |
|
|
echo "Content of unit test folder "
|
|
ls cmakebuild/${{ matrix.config.ri-unit-test-path }}
|
|
cmakebuild/${{ matrix.config.ri-unit-test-path }}
|
|
|
|
- name: (Python) Build Python module
|
|
if: matrix.config.build-python-module
|
|
shell: bash
|
|
run: |
|
|
uv pip install --system build wheel
|
|
cd GrpcInterface/Python
|
|
${{ steps.python-path.outputs.PYTHON_EXECUTABLE }} -m build
|
|
|
|
- name: (Python) Check types using mypy
|
|
if: matrix.config.build-python-module
|
|
shell: bash
|
|
run: |
|
|
uv pip install --system mypy types-protobuf
|
|
cd GrpcInterface/Python
|
|
${{ steps.python-path.outputs.PYTHON_EXECUTABLE }} -m mypy rips/*.py rips/generated/generated_classes.py
|
|
|
|
- name: Run pytest
|
|
if: matrix.config.execute-pytests
|
|
env:
|
|
RESINSIGHT_EXECUTABLE: ${{ runner.workspace }}/ResInsight/cmakebuild/ResInsight${{ runner.os == 'Windows' && '.exe' || '' }}
|
|
run: |
|
|
cd GrpcInterface/Python/rips
|
|
${{ steps.python-path.outputs.PYTHON_EXECUTABLE }} -m pytest --console
|
|
|
|
- name: Upload python distribution folder
|
|
if: matrix.config.publish-to-pypi
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: python-distribution
|
|
path: GrpcInterface/Python/dist
|
|
retention-days: 5
|
|
|
|
- name: Upload artifacts
|
|
uses: actions/upload-artifact@v7
|
|
with:
|
|
name: ResInsight-${{ matrix.config.name }}
|
|
path: ${{ runner.workspace }}/ResInsight/cmakebuild/install
|
|
retention-days: 5
|
|
|
|
pypi-publish:
|
|
name: Upload release to PyPI
|
|
needs: build
|
|
runs-on: ubuntu-latest
|
|
environment:
|
|
name: pypi
|
|
url: https://pypi.org/p/rips
|
|
permissions:
|
|
id-token: write # IMPORTANT: this permission is mandatory for trusted publishing
|
|
|
|
# If this is a tagged release
|
|
if: startsWith(github.ref, 'refs/tags')
|
|
steps:
|
|
- name: Download python distribution folder
|
|
uses: actions/download-artifact@v8
|
|
with:
|
|
name: python-distribution
|
|
path: dist
|
|
|
|
- name: Display structure of downloaded files
|
|
run: ls -R
|
|
|
|
- name: Publish Python package to pypi
|
|
# https://docs.pypi.org/trusted-publishers/adding-a-publisher/
|
|
uses: pypa/gh-action-pypi-publish@release/v1
|