From d38f3cb42f3a15f838a7376a93618a1b9ae3ace4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=CE=94=20=C7=80=20=CE=9E=20=C8=BC?= Date: Tue, 31 Oct 2023 11:17:13 +0000 Subject: [PATCH] Refactored Earthfile to improve caching Targets have been parallelized so that initial push of the container cache should build the whole build a little quicker, plus the container should now use the cache for more of the build and so speed up normal builds to just the compilation and test of the code that has changed The `build_cache` target now builds a `build-cache:latest` container that is stored in the GitLab project Container Registry, and then used (if it exists) by the `test_build` target. The `build_cache` task runs under 3 conditions, 1. the container does not exist, 2. if scheduled, 3. if run manually from the Pipelines page in the GitLab interface. It is recommended that the build is set up to run on a weekly schedule via the `Pipeline schedules` page in GitLab with the schedule of `0 2 * * 6`. --- .gitignore | 5 +- .gitlab-ci.yml | 46 ++++++--- Earthfile | 174 ++++++++++++++++++++------------- veilid-cli/Cargo.toml | 1 + veilid-core/Cargo.toml | 2 + veilid-flutter/rust/Cargo.toml | 1 + veilid-server/Cargo.toml | 1 + veilid-tools/Cargo.toml | 1 + veilid-wasm/Cargo.toml | 1 + 9 files changed, 153 insertions(+), 79 deletions(-) diff --git a/.gitignore b/.gitignore index 4d82ead2..b410b5be 100644 --- a/.gitignore +++ b/.gitignore @@ -236,4 +236,7 @@ veilid-python/demo/.demokeys .vscode/ ## Ignore emacs backup files -~* \ No newline at end of file +~* + +# Earthly temporary build output +.tmp-earthly-out/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4c975c4..6e132a10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,29 +5,53 @@ variables: GIT_SUBMODULE_STRATEGY: normal stages: + - prepare - test - build_packages - release - distribute -unit_test_linux: - stage: test +# base earthly setup for jobs +.base: + tags: [ saas-linux-medium-amd64 ] image: docker services: - docker:dind + +.earthly: &earthly_setup + - apk update && apk add git + - wget https://github.com/earthly/earthly/releases/download/v0.7.15/earthly-linux-amd64 -O /usr/local/bin/earthly + - chmod +x /usr/local/bin/earthly + - earthly bootstrap + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - test "$CI_PROJECT_PATH" != "veilid/veilid" && project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH" + +# Create the build container if: +# - no container in the registry +# - run as part of a schedule +# - run manually from the pipelines web page +build_cache: + extends: .base + stage: prepare + script: + - if ! docker manifest inspect registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest > /dev/null || [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] || [[ "$CI_PIPELINE_SOURCE" == "web" ]] ; then + - *earthly_setup + - earthly --use-inline-cache --save-inline-cache --strict --push -P +build-linux-cache $project_args + - else + - echo "No need to rebuild" + - fi + +# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry +test_build: + extends: .base + stage: test only: - main + - pushes - merge_requests - before_script: - - apk update && apk add git - - wget https://github.com/earthly/earthly/releases/download/v0.6.2/earthly-linux-amd64 -O /usr/local/bin/earthly - - chmod +x /usr/local/bin/earthly - - earthly bootstrap - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY script: - - if [ "$CI_COMMIT_BRANCH" == "main" ]; then cache_args="--save-inline-cache --push"; fi - - if [ "$CI_PROJECT_PATH" != "veilid/veilid" ]; then project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH"; fi - - earthly --use-inline-cache $cache_args --ci +unit-tests-linux $project_args + - *earthly_setup + - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args resource_group: test when: manual diff --git a/Earthfile b/Earthfile index 7a2d2e6b..53af93f5 100644 --- a/Earthfile +++ b/Earthfile @@ -1,10 +1,27 @@ -VERSION 0.6 +VERSION 0.7 + +######################################################################################################################## +## ARGUMENTS +## +## CI_PROJECT_PATH - used so that forks can refer to themselves, e.g. to use the fork's own registry cache in the +## `+build-linux-cache` target, and defaulting to `veilid/veilid` if not specified +## +## BASE - tells the build whether it should run in the default mode which runs the complete build, or run by starting +## with the remote `container` value which uses `build-cache:latest` as set up in the projects Container Registry +## +######################################################################################################################## # Start with older Ubuntu to ensure GLIBC symbol versioning support for older linux # Ensure we are using an amd64 platform because some of these targets use cross-platform tooling FROM ubuntu:18.04 +ENV ZIG_VERSION=0.11.0-dev.3978+711b4e93e +ENV RUSTUP_HOME=/usr/local/rustup +ENV CARGO_HOME=/usr/local/cargo +ENV PATH=$PATH:/usr/local/cargo/bin:/usr/local/zig +ENV LD_LIBRARY_PATH=/usr/local/lib +WORKDIR /veilid -# Install build prerequisites +# Install build prerequisites & setup required directories deps-base: RUN apt-get -y update RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev @@ -14,25 +31,27 @@ deps-capnp: FROM +deps-base COPY scripts/earthly/install_capnproto.sh / RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh + SAVE ARTIFACT /usr/local/bin/capnp* bin/ + SAVE ARTIFACT /usr/local/lib/lib* lib/ + SAVE ARTIFACT /usr/local/lib/pkgconfig lib/ # Install protoc deps-protoc: - FROM +deps-capnp + FROM +deps-base COPY scripts/earthly/install_protoc.sh / RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh + SAVE ARTIFACT /usr/local/bin/protoc bin/protoc # Install Rust deps-rust: - FROM +deps-protoc - ENV RUSTUP_HOME=/usr/local/rustup - ENV CARGO_HOME=/usr/local/cargo - ENV PATH=/usr/local/cargo/bin:$PATH - RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal + FROM +deps-base + RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \ rustup --version; \ cargo --version; \ rustc --version; - # ARM64 Linux + # Linux + RUN rustup target add x86_64-unknown-linux-gnu RUN rustup target add aarch64-unknown-linux-gnu # Android RUN rustup target add aarch64-linux-android @@ -41,20 +60,31 @@ deps-rust: RUN rustup target add x86_64-linux-android # WASM RUN rustup target add wasm32-unknown-unknown - -# Install Linux cross-platform tooling -deps-cross: - FROM +deps-rust - RUN curl https://ziglang.org/builds/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e.tar.xz | tar -C /usr/local -xJf - - RUN mv /usr/local/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e /usr/local/zig - ENV PATH=$PATH:/usr/local/zig + # Caching tool + RUN cargo install cargo-chef + # Install Linux cross-platform tooling + RUN curl -O https://ziglang.org/builds/zig-linux-x86_64-$ZIG_VERSION.tar.xz + RUN tar -C /usr/local -xJf zig-linux-x86_64-$ZIG_VERSION.tar.xz + RUN mv /usr/local/zig-linux-x86_64-$ZIG_VERSION /usr/local/zig RUN cargo install cargo-zigbuild - RUN rustup target add x86_64-unknown-linux-gnu - RUN rustup target add aarch64-unknown-linux-gnu + SAVE ARTIFACT $RUSTUP_HOME rustup + SAVE ARTIFACT $CARGO_HOME cargo + SAVE ARTIFACT /usr/local/cargo/bin/cargo-zigbuild + SAVE ARTIFACT /usr/local/zig # Install android tooling deps-android: - FROM +deps-cross + FROM +deps-base + BUILD +deps-protoc + COPY +deps-protoc/bin/* /usr/local/bin/ + BUILD +deps-capnp + COPY +deps-capnp/bin/* /usr/local/bin/ + COPY +deps-capnp/lib/* /usr/local/lib/ + BUILD +deps-rust + COPY +deps-rust/cargo /usr/local/cargo + COPY +deps-rust/rustup /usr/local/rustup + COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild + COPY +deps-rust/zig /usr/local/zig RUN apt-get install -y openjdk-9-jdk-headless RUN mkdir /Android; mkdir /Android/Sdk RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-9123335_latest.zip @@ -62,18 +92,51 @@ deps-android: RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33 cmdline-tools\;latest RUN rm -rf /Android/cmdline-tools RUN apt-get clean - + # Just linux build not android deps-linux: - FROM +deps-cross - RUN apt-get clean + FROM +deps-base + BUILD +deps-protoc + COPY +deps-protoc/bin/* /usr/local/bin/ + BUILD +deps-capnp + COPY +deps-capnp/bin/* /usr/local/bin/ + COPY +deps-capnp/lib/* /usr/local/lib/ + BUILD +deps-rust + COPY +deps-rust/cargo /usr/local/cargo + COPY +deps-rust/rustup /usr/local/rustup + COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild + COPY +deps-rust/zig /usr/local/zig -# Code + Linux deps -code-linux: +build-linux-cache: FROM +deps-linux + RUN mkdir veilid-cli veilid-core veilid-server veilid-tools veilid-wasm veilid-flutter veilid-flutter/rust + COPY --dir .cargo scripts Cargo.lock Cargo.toml . + COPY veilid-cli/Cargo.toml veilid-cli + COPY veilid-core/Cargo.toml veilid-core + COPY veilid-server/Cargo.toml veilid-server + COPY veilid-tools/Cargo.toml veilid-tools + COPY veilid-flutter/rust/Cargo.lock veilid-flutter/rust/Cargo.toml veilid-flutter/rust + COPY veilid-wasm/Cargo.toml veilid-wasm + RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> .cargo/config.toml + RUN cargo chef prepare --recipe-path recipe.json + RUN cargo chef cook --recipe-path recipe.json + RUN echo $PROJECT_PATH + SAVE ARTIFACT target + ARG CI_PROJECT_PATH=veilid/veilid + SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest + +code-linux: + # This target will either use the full earthly cache of local use (+build-linux-cache), or will use a containerized + # version of the +build-linux-cache from the registry + ARG BASE=local + IF [ "$BASE" = "local" ] + FROM +build-linux-cache + ELSE + ARG CI_PROJECT_PATH=veilid/veilid + FROM registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest + # FROM registry.gitlab.com/veilid/build-cache:latest + END COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid - RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml - WORKDIR /veilid # Code + Linux + Android deps code-android: @@ -81,89 +144,68 @@ code-android: COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml - WORKDIR /veilid # Clippy only clippy: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid RUN cargo clippy - SAVE IMAGE --push registry.gitl ab.com/$CI_PROJECT_PATH/clippy:latest # Build build-release: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid RUN cargo build --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE ARTIFACT ./target/release AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-release:latest + SAVE ARTIFACT ./target/release AS LOCAL ./target/release build: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid RUN cargo build -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE ARTIFACT ./target/debug AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build:latest + SAVE ARTIFACT ./target/debug AS LOCAL ./target/debug build-linux-amd64: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid - RUN cargo build --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core + RUN cargo zigbuild --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64:latest build-linux-amd64-debug: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid - RUN cargo build --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core + RUN cargo zigbuild --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64-debug:latest build-linux-arm64: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid RUN cargo zigbuild --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core SAVE ARTIFACT ./target/aarch64-unknown-linux-gnu AS LOCAL ./target/artifacts/aarch64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-arm64:latest -# build-android: -# FROM +code-android -# ARG CI_PROJECT_PATH=veilid/veilid -# WORKDIR /veilid/veilid-core -# ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/ -# RUN cargo build --target aarch64-linux-android --release -# RUN cargo build --target armv7-linux-androideabi --release -# RUN cargo build --target i686-linux-android --release -# RUN cargo build --target x86_64-linux-android --release -# WORKDIR /veilid -# SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android -# SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi -# SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android -# SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android -# SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-android:latest +build-android: + FROM +code-android + WORKDIR /veilid/veilid-core + ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/ + RUN cargo build --target aarch64-linux-android --release + RUN cargo build --target armv7-linux-androideabi --release + RUN cargo build --target i686-linux-android --release + RUN cargo build --target x86_64-linux-android --release + WORKDIR /veilid + SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android + SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi + SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android + SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android # Unit tests unit-tests-linux: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid ENV RUST_BACKTRACE=1 - RUN cargo test --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux:latest + RUN cargo test -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core # TODO: Change t0 cross so that they work on any platform unit-tests-linux-amd64: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid ENV RUST_BACKTRACE=1 RUN cargo test --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-amd64:latest unit-tests-linux-arm64: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid ENV RUST_BACKTRACE=1 RUN cargo test --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-arm64:latest # Package package-linux-amd64-deb: @@ -232,8 +274,6 @@ package-linux-arm64-rpm: # save artifacts SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/ - - package-linux-amd64: BUILD +package-linux-amd64-deb BUILD +package-linux-amd64-rpm diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index ee824b44..92418e7b 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -6,6 +6,7 @@ version = "0.2.3" authors = ["Veilid Team "] edition = "2021" license = "MPL-2.0" +resolver = "2" [[bin]] name = "veilid-cli" diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 783d4ead..f32a2b2c 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -8,9 +8,11 @@ authors = ["Veilid Team "] edition = "2021" build = "build.rs" license = "MPL-2.0" +resolver = "2" [lib] crate-type = ["cdylib", "staticlib", "rlib"] +path = "src/lib.rs" [features] diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index 11fb0f05..a7accaa5 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" [lib] crate-type = ["cdylib", "staticlib", "rlib"] +path = "src/lib.rs" [features] default = ["rt-tokio", "veilid-core/default"] diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index cedb3834..cd1845b6 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -7,6 +7,7 @@ description = "Veilid Server" authors = ["Veilid Team "] license = "MPL-2.0" edition = "2021" +resolver = "2" [[bin]] name = "veilid-server" diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index ea8fd23e..cb049044 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" [lib] # staticlib for iOS tests, cydlib for android tests, rlib for everything else crate-type = ["cdylib", "staticlib", "rlib"] +path = "src/lib.rs" [features] default = ["rt-tokio"] diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index 63d98e90..79292ba6 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" [lib] crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" [features] default = ["veilid-core/default-wasm"]