Refactored Earthfile to improve caching

Targets have been parallelized so that initial push of the container cache should build the whole build a little quicker, plus the container should now use the cache for more of the build and so speed up normal builds to just the compilation and test of the code that has changed

The `build_cache` target now builds a `build-cache:latest` container that is stored in the GitLab project Container Registry, and then used (if it exists) by the `test_build` target.  The `build_cache` task runs under 3 conditions, 1. the container does not exist, 2. if scheduled, 3. if run manually from the Pipelines page in the GitLab interface.

It is recommended that the build is set up to run on a weekly schedule via the `Pipeline schedules` page in GitLab with the schedule of `0 2 * * 6`.

.gitignore

@@ -236,4 +236,7 @@ veilid-python/demo/.demokeys
 .vscode/
 
 ## Ignore emacs backup files
-~*
+~*
+
+# Earthly temporary build output
+.tmp-earthly-out/

.gitlab-ci.yml

@@ -5,29 +5,53 @@ variables:
   GIT_SUBMODULE_STRATEGY: normal
 
 stages:
+  - prepare
   - test
   - build_packages
   - release
   - distribute
 
-unit_test_linux:
+# base earthly setup for jobs
-  stage: test
+.base:
+  tags: [ saas-linux-medium-amd64 ]
   image: docker
   services:
     - docker:dind
+
+.earthly: &earthly_setup
+  - apk update && apk add git
+  - wget https://github.com/earthly/earthly/releases/download/v0.7.15/earthly-linux-amd64 -O /usr/local/bin/earthly
+  - chmod +x /usr/local/bin/earthly
+  - earthly bootstrap
+  - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
+  - test "$CI_PROJECT_PATH" != "veilid/veilid" && project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH"
+
+# Create the build container if:
+#  - no container in the registry
+#  - run as part of a schedule
+#  - run manually from the pipelines web page
+build_cache:
+  extends: .base
+  stage: prepare
+  script:
+    - if ! docker manifest inspect registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest > /dev/null || [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] || [[ "$CI_PIPELINE_SOURCE" == "web" ]] ; then
+    -   *earthly_setup
+    -   earthly --use-inline-cache --save-inline-cache --strict --push -P +build-linux-cache $project_args
+    - else
+    -   echo "No need to rebuild"
+    - fi
+
+# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry
+test_build:
+  extends: .base
+  stage: test
   only:
     - main
+    - pushes
     - merge_requests
-  before_script:
-    - apk update && apk add git
-    - wget https://github.com/earthly/earthly/releases/download/v0.6.2/earthly-linux-amd64 -O /usr/local/bin/earthly
-    - chmod +x /usr/local/bin/earthly
-    - earthly bootstrap
-    - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY
   script:
-    - if [ "$CI_COMMIT_BRANCH" == "main" ]; then cache_args="--save-inline-cache --push"; fi
+    - *earthly_setup
-    - if [ "$CI_PROJECT_PATH" != "veilid/veilid" ]; then project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH"; fi
+    - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args
-    - earthly --use-inline-cache $cache_args --ci +unit-tests-linux $project_args
   resource_group: test
   when: manual
 

Earthfile

@@ -1,10 +1,27 @@
-VERSION 0.6
+VERSION 0.7
+
+########################################################################################################################
+## ARGUMENTS
+##
+## CI_PROJECT_PATH - used so that forks can refer to themselves, e.g. to use the fork's own registry cache in the
+## `+build-linux-cache` target, and defaulting to `veilid/veilid` if not specified
+##
+## BASE - tells the build whether it should run in the default mode which runs the complete build, or run by starting
+## with the remote `container` value which uses `build-cache:latest` as set up in the projects Container Registry
+##
+########################################################################################################################
 
 # Start with older Ubuntu to ensure GLIBC symbol versioning support for older linux
 # Ensure we are using an amd64 platform because some of these targets use cross-platform tooling
 FROM ubuntu:18.04
+ENV ZIG_VERSION=0.11.0-dev.3978+711b4e93e
+ENV RUSTUP_HOME=/usr/local/rustup
+ENV CARGO_HOME=/usr/local/cargo
+ENV PATH=$PATH:/usr/local/cargo/bin:/usr/local/zig
+ENV LD_LIBRARY_PATH=/usr/local/lib
+WORKDIR /veilid
 
-# Install build prerequisites
+# Install build prerequisites & setup required directories
 deps-base:
     RUN apt-get -y update
     RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev
@@ -14,25 +31,27 @@ deps-capnp:
     FROM +deps-base
     COPY scripts/earthly/install_capnproto.sh /
     RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh
+    SAVE ARTIFACT /usr/local/bin/capnp* bin/
+    SAVE ARTIFACT /usr/local/lib/lib* lib/
+    SAVE ARTIFACT /usr/local/lib/pkgconfig lib/
 
 # Install protoc
 deps-protoc:
-    FROM +deps-capnp
+    FROM +deps-base
     COPY scripts/earthly/install_protoc.sh /
     RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh
+    SAVE ARTIFACT /usr/local/bin/protoc bin/protoc
 
 # Install Rust
 deps-rust:
-    FROM +deps-protoc
+    FROM +deps-base
-    ENV RUSTUP_HOME=/usr/local/rustup
+    RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal
-    ENV CARGO_HOME=/usr/local/cargo
-    ENV PATH=/usr/local/cargo/bin:$PATH
-    RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal 
     RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
         rustup --version; \
         cargo --version; \
         rustc --version;
-    # ARM64 Linux
+    # Linux
+    RUN rustup target add x86_64-unknown-linux-gnu
     RUN rustup target add aarch64-unknown-linux-gnu
     # Android
     RUN rustup target add aarch64-linux-android
@@ -41,20 +60,31 @@ deps-rust:
     RUN rustup target add x86_64-linux-android
     # WASM
     RUN rustup target add wasm32-unknown-unknown
-
+    # Caching tool
-# Install Linux cross-platform tooling
+    RUN cargo install cargo-chef
-deps-cross:
+    # Install Linux cross-platform tooling
-    FROM +deps-rust
+    RUN curl -O https://ziglang.org/builds/zig-linux-x86_64-$ZIG_VERSION.tar.xz
-    RUN curl https://ziglang.org/builds/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e.tar.xz | tar -C /usr/local -xJf -
+    RUN tar -C /usr/local -xJf zig-linux-x86_64-$ZIG_VERSION.tar.xz
-    RUN mv /usr/local/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e /usr/local/zig
+    RUN mv /usr/local/zig-linux-x86_64-$ZIG_VERSION /usr/local/zig
-    ENV PATH=$PATH:/usr/local/zig
     RUN cargo install cargo-zigbuild
-    RUN rustup target add x86_64-unknown-linux-gnu
+    SAVE ARTIFACT $RUSTUP_HOME rustup
-    RUN rustup target add aarch64-unknown-linux-gnu
+    SAVE ARTIFACT $CARGO_HOME cargo
+    SAVE ARTIFACT /usr/local/cargo/bin/cargo-zigbuild
+    SAVE ARTIFACT /usr/local/zig
 
 # Install android tooling
 deps-android:
-    FROM +deps-cross
+    FROM +deps-base
+    BUILD +deps-protoc
+    COPY +deps-protoc/bin/* /usr/local/bin/
+    BUILD +deps-capnp
+    COPY +deps-capnp/bin/* /usr/local/bin/
+    COPY +deps-capnp/lib/* /usr/local/lib/
+    BUILD +deps-rust
+    COPY +deps-rust/cargo /usr/local/cargo
+    COPY +deps-rust/rustup /usr/local/rustup
+    COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild
+    COPY +deps-rust/zig /usr/local/zig
     RUN apt-get install -y openjdk-9-jdk-headless
     RUN mkdir /Android; mkdir /Android/Sdk
     RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-9123335_latest.zip
@@ -62,18 +92,51 @@ deps-android:
     RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33 cmdline-tools\;latest
     RUN rm -rf /Android/cmdline-tools
     RUN apt-get clean
-    
+
 # Just linux build not android
 deps-linux:
-    FROM +deps-cross
+    FROM +deps-base
-    RUN apt-get clean
+    BUILD +deps-protoc
+    COPY +deps-protoc/bin/* /usr/local/bin/
+    BUILD +deps-capnp
+    COPY +deps-capnp/bin/* /usr/local/bin/
+    COPY +deps-capnp/lib/* /usr/local/lib/
+    BUILD +deps-rust
+    COPY +deps-rust/cargo /usr/local/cargo
+    COPY +deps-rust/rustup /usr/local/rustup
+    COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild
+    COPY +deps-rust/zig /usr/local/zig
 
-# Code + Linux deps
+build-linux-cache:
-code-linux:
     FROM +deps-linux
+    RUN mkdir veilid-cli veilid-core veilid-server veilid-tools veilid-wasm veilid-flutter veilid-flutter/rust
+    COPY --dir .cargo scripts Cargo.lock Cargo.toml .
+    COPY veilid-cli/Cargo.toml veilid-cli
+    COPY veilid-core/Cargo.toml veilid-core
+    COPY veilid-server/Cargo.toml veilid-server
+    COPY veilid-tools/Cargo.toml veilid-tools
+    COPY veilid-flutter/rust/Cargo.lock veilid-flutter/rust/Cargo.toml veilid-flutter/rust
+    COPY veilid-wasm/Cargo.toml veilid-wasm
+    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> .cargo/config.toml
+    RUN cargo chef prepare --recipe-path recipe.json
+    RUN cargo chef cook --recipe-path recipe.json
+    RUN echo $PROJECT_PATH
+    SAVE ARTIFACT target
+    ARG CI_PROJECT_PATH=veilid/veilid
+    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest
+
+code-linux:
+    # This target will either use the full earthly cache of local use (+build-linux-cache), or will use a containerized
+    # version of the +build-linux-cache from the registry
+    ARG BASE=local
+    IF [ "$BASE" = "local" ]
+        FROM +build-linux-cache
+    ELSE
+        ARG CI_PROJECT_PATH=veilid/veilid
+        FROM registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest
+        # FROM registry.gitlab.com/veilid/build-cache:latest
+    END
     COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
-    RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
-    WORKDIR /veilid
 
 # Code + Linux + Android deps
 code-android:
@@ -81,89 +144,68 @@ code-android:
     COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
     RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
     RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml
-    WORKDIR /veilid
 
 # Clippy only
 clippy:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
     RUN cargo clippy
-    SAVE IMAGE --push registry.gitl ab.com/$CI_PROJECT_PATH/clippy:latest
 
 # Build
 build-release:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
     RUN cargo build --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE ARTIFACT ./target/release AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
+    SAVE ARTIFACT ./target/release AS LOCAL ./target/release
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-release:latest
 
 build:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
     RUN cargo build -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE ARTIFACT ./target/debug AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
+    SAVE ARTIFACT ./target/debug AS LOCAL ./target/debug
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build:latest
 
 build-linux-amd64:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
+    RUN cargo zigbuild --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    RUN cargo build --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
     SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64:latest
 
 build-linux-amd64-debug:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
+    RUN cargo zigbuild --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    RUN cargo build --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
     SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64-debug:latest
 
 build-linux-arm64:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
     RUN cargo zigbuild --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
     SAVE ARTIFACT ./target/aarch64-unknown-linux-gnu AS LOCAL ./target/artifacts/aarch64-unknown-linux-gnu
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-arm64:latest
 
-# build-android:
+build-android:
-#     FROM +code-android
+    FROM +code-android
-#     ARG CI_PROJECT_PATH=veilid/veilid
+    WORKDIR /veilid/veilid-core
-#     WORKDIR /veilid/veilid-core
+    ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/
-#     ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/
+    RUN cargo build --target aarch64-linux-android --release
-#     RUN cargo build --target aarch64-linux-android --release
+    RUN cargo build --target armv7-linux-androideabi --release
-#     RUN cargo build --target armv7-linux-androideabi --release
+    RUN cargo build --target i686-linux-android --release
-#     RUN cargo build --target i686-linux-android --release
+    RUN cargo build --target x86_64-linux-android --release
-#     RUN cargo build --target x86_64-linux-android --release
+    WORKDIR /veilid
-#     WORKDIR /veilid
+    SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android
-#     SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android
+    SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi
-#     SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi
+    SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android
-#     SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android
+    SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android
-#     SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android
-#     SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-android:latest
 
 # Unit tests
 unit-tests-linux:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
     ENV RUST_BACKTRACE=1
-    RUN cargo test --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
+    RUN cargo test -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux:latest
 
 # TODO: Change t0 cross so that they work on any platform
 unit-tests-linux-amd64:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
     ENV RUST_BACKTRACE=1
     RUN cargo test --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-amd64:latest
 
 unit-tests-linux-arm64:
     FROM +code-linux
-    ARG CI_PROJECT_PATH=veilid/veilid
     ENV RUST_BACKTRACE=1
     RUN cargo test --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
-    SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-arm64:latest
 
 # Package 
 package-linux-amd64-deb:
@@ -232,8 +274,6 @@ package-linux-arm64-rpm:
     # save artifacts
     SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/
 
-
-
 package-linux-amd64:
     BUILD +package-linux-amd64-deb
     BUILD +package-linux-amd64-rpm

veilid-cli/Cargo.toml

@@ -6,6 +6,7 @@ version = "0.2.3"
 authors = ["Veilid Team <contact@veilid.com>"]
 edition = "2021"
 license = "MPL-2.0"
+resolver = "2"
 
 [[bin]]
 name = "veilid-cli"

veilid-core/Cargo.toml

@@ -8,9 +8,11 @@ authors = ["Veilid Team <contact@veilid.com>"]
 edition = "2021"
 build = "build.rs"
 license = "MPL-2.0"
+resolver = "2"
 
 [lib]
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"
 
 [features]
 

veilid-flutter/rust/Cargo.toml

@@ -9,6 +9,7 @@ edition = "2021"
 
 [lib]
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"
 
 [features]
 default = ["rt-tokio", "veilid-core/default"]

veilid-server/Cargo.toml

@@ -7,6 +7,7 @@ description = "Veilid Server"
 authors = ["Veilid Team <contact@veilid.com>"]
 license = "MPL-2.0"
 edition = "2021"
+resolver = "2"
 
 [[bin]]
 name = "veilid-server"

veilid-tools/Cargo.toml

@@ -11,6 +11,7 @@ edition = "2021"
 [lib]
 # staticlib for iOS tests, cydlib for android tests, rlib for everything else
 crate-type = ["cdylib", "staticlib", "rlib"]
+path = "src/lib.rs"
 
 [features]
 default = ["rt-tokio"]

veilid-wasm/Cargo.toml

@@ -9,6 +9,7 @@ edition = "2021"
 
 [lib]
 crate-type = ["cdylib", "rlib"]
+path = "src/lib.rs"
 
 [features]
 default = ["veilid-core/default-wasm"]