Refactored Earthfile to improve caching

Targets have been parallelized so that initial push of the container cache should build the whole build a little quicker, plus the container should now use the cache for more of the build and so speed up normal builds to just the compilation and test of the code that has changed The `build_cache` target now builds a `build-cache:latest` container that is stored in the GitLab project Container Registry, and then used (if it exists) by the `test_build` target. The `build_cache` task runs under 3 conditions, 1. the container does not exist, 2. if scheduled, 3. if run manually from the Pipelines page in the GitLab interface. It is recommended that the build is set up to run on a weekly schedule via the `Pipeline schedules` page in GitLab with the schedule of `0 2 * * 6`.
2024-12-23 14:40:02 -06:00 · 2023-10-31 11:17:13 +00:00 · 2023-10-31 11:17:13 +00:00 · d38f3cb42f
commit d38f3cb42f
parent ff56634667
9 changed files with 153 additions and 79 deletions
--- a/.gitignore
+++ b/.gitignore
@ -236,4 +236,7 @@ veilid-python/demo/.demokeys
 .vscode/

 ## Ignore emacs backup files
-~*
+~*
+
+# Earthly temporary build output
+.tmp-earthly-out/
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -5,29 +5,53 @@ variables:
  GIT_SUBMODULE_STRATEGY: normal

 stages:
+  - prepare
  - test
  - build_packages
  - release
  - distribute

-unit_test_linux:
-  stage: test
+# base earthly setup for jobs
+.base:
+  tags: [ saas-linux-medium-amd64 ]
  image: docker
  services:
    - docker:dind
+
+.earthly: &earthly_setup
+  - apk update && apk add git
+  - wget https://github.com/earthly/earthly/releases/download/v0.7.15/earthly-linux-amd64 -O /usr/local/bin/earthly
+  - chmod +x /usr/local/bin/earthly
+  - earthly bootstrap
+  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+  - test "$CI_PROJECT_PATH" != "veilid/veilid" && project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH"
+
+# Create the build container if:
+#  - no container in the registry
+#  - run as part of a schedule
+#  - run manually from the pipelines web page
+build_cache:
+  extends: .base
+  stage: prepare
+  script:
+    - if ! docker manifest inspect registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest > /dev/null || [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] || [[ "$CI_PIPELINE_SOURCE" == "web" ]] ; then
+    -   *earthly_setup
+    -   earthly --use-inline-cache --save-inline-cache --strict --push -P +build-linux-cache $project_args
+    - else
+    -   echo "No need to rebuild"
+    - fi
+
+# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry
+test_build:
+  extends: .base
+  stage: test
  only:
    - main
+    - pushes
    - merge_requests
-  before_script:
-    - apk update && apk add git
-    - wget https://github.com/earthly/earthly/releases/download/v0.6.2/earthly-linux-amd64 -O /usr/local/bin/earthly
-    - chmod +x /usr/local/bin/earthly
-    - earthly bootstrap
-    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
  script:
-    - if [ "$CI_COMMIT_BRANCH" == "main" ]; then cache_args="--save-inline-cache --push"; fi
-    - if [ "$CI_PROJECT_PATH" != "veilid/veilid" ]; then project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH"; fi
-    - earthly --use-inline-cache $cache_args --ci +unit-tests-linux $project_args
+    - *earthly_setup
+    - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args
  resource_group: test
  when: manual

--- a/174
+++ b/174
@ -1,10 +1,27 @@
-VERSION 0.6
+VERSION 0.7
+
+########################################################################################################################
+## ARGUMENTS
+##
+## CI_PROJECT_PATH - used so that forks can refer to themselves, e.g. to use the fork's own registry cache in the
+## `+build-linux-cache` target, and defaulting to `veilid/veilid` if not specified
+##
+## BASE - tells the build whether it should run in the default mode which runs the complete build, or run by starting
+## with the remote `container` value which uses `build-cache:latest` as set up in the projects Container Registry
+##
+########################################################################################################################

 # Start with older Ubuntu to ensure GLIBC symbol versioning support for older linux
 # Ensure we are using an amd64 platform because some of these targets use cross-platform tooling
 FROM ubuntu:18.04
+ENV ZIG_VERSION=0.11.0-dev.3978+711b4e93e
+ENV RUSTUP_HOME=/usr/local/rustup
+ENV CARGO_HOME=/usr/local/cargo
+ENV PATH=$PATH:/usr/local/cargo/bin:/usr/local/zig
+ENV LD_LIBRARY_PATH=/usr/local/lib
+WORKDIR /veilid

-# Install build prerequisites
+# Install build prerequisites & setup required directories
 deps-base:
    RUN apt-get -y update
    RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev
@ -14,25 +31,27 @@ deps-capnp:
    FROM +deps-base
    COPY scripts/earthly/install_capnproto.sh /
    RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh
+    SAVE ARTIFACT /usr/local/bin/capnp* bin/
+    SAVE ARTIFACT /usr/local/lib/lib* lib/
+    SAVE ARTIFACT /usr/local/lib/pkgconfig lib/

 # Install protoc
 deps-protoc:
-    FROM +deps-capnp
+    FROM +deps-base
    COPY scripts/earthly/install_protoc.sh /
    RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh
+    SAVE ARTIFACT /usr/local/bin/protoc bin/protoc

 # Install Rust
 deps-rust:
-    FROM +deps-protoc
-    ENV RUSTUP_HOME=/usr/local/rustup
-    ENV CARGO_HOME=/usr/local/cargo
-    ENV PATH=/usr/local/cargo/bin:$PATH
-    RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal 
+    FROM +deps-base
+    RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal
    RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
        rustup --version; \
        cargo --version; \
        rustc --version;
-    # ARM64 Linux
+    # Linux
+    RUN rustup target add x86_64-unknown-linux-gnu
    RUN rustup target add aarch64-unknown-linux-gnu
    # Android
    RUN rustup target add aarch64-linux-android
@ -41,20 +60,31 @@ deps-rust:
    RUN rustup target add x86_64-linux-android
    # WASM
    RUN rustup target add wasm32-unknown-unknown
-
-# Install Linux cross-platform tooling
-deps-cross:
-    FROM +deps-rust
-    RUN curl https://ziglang.org/builds/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e.tar.xz | tar -C /usr/local -xJf -
-    RUN mv /usr/local/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e /usr/local/zig
-    ENV PATH=$PATH:/usr/local/zig
+    # Caching tool
+    RUN cargo install cargo-chef
+    # Install Linux cross-platform tooling
+    RUN curl -O https://ziglang.org/builds/zig-linux-x86_64-$ZIG_VERSION.tar.xz
+    RUN tar -C /usr/local -xJf zig-linux-x86_64-$ZIG_VERSION.tar.xz
+    RUN mv /usr/local/zig-linux-x86_64-$ZIG_VERSION /usr/local/zig
    RUN cargo install cargo-zigbuild
-    RUN rustup target add x86_64-unknown-linux-gnu
-    RUN rustup target add aarch64-unknown-linux-gnu
+    SAVE ARTIFACT $RUSTUP_HOME rustup
+    SAVE ARTIFACT $CARGO_HOME cargo
+    SAVE ARTIFACT /usr/local/cargo/bin/cargo-zigbuild
+    SAVE ARTIFACT /usr/local/zig

 # Install android tooling
 deps-android:
-    FROM +deps-cross
+    FROM +deps-base
+    BUILD +deps-protoc
+    COPY +deps-protoc/bin/* /usr/local/bin/
+    BUILD +deps-capnp
+    COPY +deps-capnp/bin/* /usr/local/bin/
+    COPY +deps-capnp/lib/* /usr/local/lib/
+    BUILD +deps-rust
+    COPY +deps-rust/cargo /usr/local/cargo
+    COPY +deps-rust/rustup /usr/local/rustup
+    COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild
+    COPY +deps-rust/zig /usr/local/zig
    RUN apt-get install -y openjdk-9-jdk-headless
    RUN mkdir /Android; mkdir /Android/Sdk
    RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-9123335_latest.zip
@ -62,18 +92,51 @@ deps-android:
    RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33 cmdline-tools\;latest
    RUN rm -rf /Android/cmdline-tools
    RUN apt-get clean
-    
+
 # Just linux build not android
 deps-linux:
-    FROM +deps-cross
-    RUN apt-get clean
+    FROM +deps-base
+    BUILD +deps-protoc
+    COPY +deps-protoc/bin/* /usr/local/bin/
+    BUILD +deps-capnp
+    COPY +deps-capnp/bin/* /usr/local/bin/
+    COPY +deps-capnp/lib/* /usr/local/lib/
+    BUILD +deps-rust
+    COPY +deps-rust/cargo /usr/local/cargo
+    COPY +deps-rust/rustup /usr/local/rustup
+    COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild
+    COPY +deps-rust/zig /usr/local/zig

-# Code + Linux deps
-code-linux:
+build-linux-cache:
    FROM +deps-linux
+    RUN mkdir veilid-cli veilid-core veilid-server veilid-tools veilid-wasm veilid-flutter veilid-flutter/rust
+    COPY --dir .cargo scripts Cargo.lock Cargo.toml .
+    COPY veilid-cli/Cargo.toml veilid-cli
+    COPY veilid-core/Cargo.toml veilid-core
+    COPY veilid-server/Cargo.toml veilid-server
+    COPY veilid-tools/Cargo.toml veilid-tools
+    COPY veilid-flutter/rust/Cargo.lock veilid-flutter/rust/Cargo.toml veilid-flutter/rust
+    COPY veilid-wasm/Cargo.toml veilid-wasm
+    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> .cargo/config.toml
+    RUN cargo chef prepare --recipe-path recipe.json
+    RUN cargo chef cook --recipe-path recipe.json
+    RUN echo $PROJECT_PATH
+    SAVE ARTIFACT target
+    ARG CI_PROJECT_PATH=veilid/veilid
+    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest
+
+code-linux:
+    # This target will either use the full earthly cache of local use (+build-linux-cache), or will use a containerized
+    # version of the +build-linux-cache from the registry
+    ARG BASE=local
+    IF [ "$BASE" = "local" ]
+        FROM +build-linux-cache
+    ELSE
+        ARG CI_PROJECT_PATH=veilid/veilid
+        FROM registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest
+        # FROM registry.gitlab.com/veilid/build-cache:latest
+    END
    COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
-    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
-    WORKDIR /veilid

 # Code + Linux + Android deps
 code-android:
@ -81,89 +144,68 @@ code-android:
    COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
    RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml
-    WORKDIR /veilid

 # Clippy only
 clippy:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    RUN cargo clippy
-    SAVE IMAGE --push registry.gitl ab.com/$CI_PROJECT_PATH/clippy:latest

 # Build
 build-release:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    RUN cargo build --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE ARTIFACT ./target/release AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-release:latest
+    SAVE ARTIFACT ./target/release AS LOCAL ./target/release

 build:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    RUN cargo build -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE ARTIFACT ./target/debug AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build:latest
+    SAVE ARTIFACT ./target/debug AS LOCAL ./target/debug

 build-linux-amd64:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
-    RUN cargo build --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
+    RUN cargo zigbuild --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
    SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64:latest

 build-linux-amd64-debug:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
-    RUN cargo build --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
+    RUN cargo zigbuild --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
    SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64-debug:latest

 build-linux-arm64:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    RUN cargo zigbuild --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
    SAVE ARTIFACT ./target/aarch64-unknown-linux-gnu AS LOCAL ./target/artifacts/aarch64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-arm64:latest

-# build-android:
-#     FROM +code-android
-#     ARG CI_PROJECT_PATH=veilid/veilid
-#     WORKDIR /veilid/veilid-core
-#     ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/
-#     RUN cargo build --target aarch64-linux-android --release
-#     RUN cargo build --target armv7-linux-androideabi --release
-#     RUN cargo build --target i686-linux-android --release
-#     RUN cargo build --target x86_64-linux-android --release
-#     WORKDIR /veilid
-#     SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android
-#     SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi
-#     SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android
-#     SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android
-#     SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-android:latest
+build-android:
+    FROM +code-android
+    WORKDIR /veilid/veilid-core
+    ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/
+    RUN cargo build --target aarch64-linux-android --release
+    RUN cargo build --target armv7-linux-androideabi --release
+    RUN cargo build --target i686-linux-android --release
+    RUN cargo build --target x86_64-linux-android --release
+    WORKDIR /veilid
+    SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android
+    SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi
+    SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android
+    SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android

 # Unit tests
 unit-tests-linux:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    ENV RUST_BACKTRACE=1
-    RUN cargo test --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux:latest
+    RUN cargo test -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core

 # TODO: Change t0 cross so that they work on any platform
 unit-tests-linux-amd64:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    ENV RUST_BACKTRACE=1
    RUN cargo test --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-amd64:latest

 unit-tests-linux-arm64:
    FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
    ENV RUST_BACKTRACE=1
    RUN cargo test --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-arm64:latest

 # Package 
 package-linux-amd64-deb:
@ -232,8 +274,6 @@ package-linux-arm64-rpm:
    # save artifacts
    SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/

-
-
 package-linux-amd64:
    BUILD +package-linux-amd64-deb
    BUILD +package-linux-amd64-rpm
--- a/veilid-cli/Cargo.toml
+++ b/veilid-cli/Cargo.toml
@ -6,6 +6,7 @@ version = "0.2.3"
 authors = ["Veilid Team <contact@veilid.com>"]
 edition = "2021"
 license = "MPL-2.0"
+resolver = "2"

 [[bin]]
 name = "veilid-cli"
--- a/veilid-core/Cargo.toml
+++ b/veilid-core/Cargo.toml
@ -8,9 +8,11 @@ authors = ["Veilid Team <contact@veilid.com>"]
 edition = "2021"
 build = "build.rs"
 license = "MPL-2.0"
+resolver = "2"

 [lib]
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"

 [features]

--- a/veilid-flutter/rust/Cargo.toml
+++ b/veilid-flutter/rust/Cargo.toml
@ -9,6 +9,7 @@ edition = "2021"

 [lib]
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"

 [features]
 default = ["rt-tokio", "veilid-core/default"]
--- a/veilid-server/Cargo.toml
+++ b/veilid-server/Cargo.toml
@ -7,6 +7,7 @@ description = "Veilid Server"
 authors = ["Veilid Team <contact@veilid.com>"]
 license = "MPL-2.0"
 edition = "2021"
+resolver = "2"

 [[bin]]
 name = "veilid-server"
--- a/veilid-tools/Cargo.toml
+++ b/veilid-tools/Cargo.toml
@ -11,6 +11,7 @@ edition = "2021"
 [lib]
 # staticlib for iOS tests, cydlib for android tests, rlib for everything else
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"

 [features]
 default = ["rt-tokio"]
--- a/veilid-wasm/Cargo.toml
+++ b/veilid-wasm/Cargo.toml
@ -9,6 +9,7 @@ edition = "2021"

 [lib]
 crate-type = ["cdylib", "rlib"]
+path = "src/lib.rs"

 [features]
 default = ["veilid-core/default-wasm"]