IntenseWebs/veilid
3
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2024-11-22 08:56:58 -06:00
Code Issues Packages Projects Releases Wiki Activity
dcd4c0d22a
veilid/Earthfile
Δ ǀ Ξ ȼ d38f3cb42f
Refactored Earthfile to improve caching 
Targets have been parallelized so that initial push of the container cache should build the whole build a little quicker, plus the container should now use the cache for more of the build and so speed up normal builds to just the compilation and test of the code that has changed

The `build_cache` target now builds a `build-cache:latest` container that is stored in the GitLab project Container Registry, and then used (if it exists) by the `test_build` target.  The `build_cache` task runs under 3 conditions, 1. the container does not exist, 2. if scheduled, 3. if run manually from the Pipelines page in the GitLab interface.

It is recommended that the build is set up to run on a weekly schedule via the `Pipeline schedules` page in GitLab with the schedule of `0 2 * * 6`.
2023-10-31 11:17:13 +00:00

288 lines
12 KiB
Plaintext
Raw Blame History

VERSION 0.7
########################################################################################################################
## ARGUMENTS
##
## CI_PROJECT_PATH - used so that forks can refer to themselves, e.g. to use the fork's own registry cache in the
## `+build-linux-cache` target, and defaulting to `veilid/veilid` if not specified
##
## BASE - tells the build whether it should run in the default mode which runs the complete build, or run by starting
## with the remote `container` value which uses `build-cache:latest` as set up in the projects Container Registry
##
########################################################################################################################
# Start with older Ubuntu to ensure GLIBC symbol versioning support for older linux
# Ensure we are using an amd64 platform because some of these targets use cross-platform tooling
FROM ubuntu:18.04
ENV ZIG_VERSION=0.11.0-dev.3978+711b4e93e
ENV RUSTUP_HOME=/usr/local/rustup
ENV CARGO_HOME=/usr/local/cargo
ENV PATH=$PATH:/usr/local/cargo/bin:/usr/local/zig
ENV LD_LIBRARY_PATH=/usr/local/lib
WORKDIR /veilid
# Install build prerequisites & setup required directories
deps-base:
RUN apt-get -y update
RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev
# Install Cap'n Proto
deps-capnp:
FROM +deps-base
COPY scripts/earthly/install_capnproto.sh /
RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh
SAVE ARTIFACT /usr/local/bin/capnp* bin/
SAVE ARTIFACT /usr/local/lib/lib* lib/
SAVE ARTIFACT /usr/local/lib/pkgconfig lib/
# Install protoc
deps-protoc:
FROM +deps-base
COPY scripts/earthly/install_protoc.sh /
RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh
SAVE ARTIFACT /usr/local/bin/protoc bin/protoc
# Install Rust
deps-rust:
FROM +deps-base
RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal
RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \
rustup --version; \
cargo --version; \
rustc --version;
# Linux
RUN rustup target add x86_64-unknown-linux-gnu
RUN rustup target add aarch64-unknown-linux-gnu
# Android
RUN rustup target add aarch64-linux-android
RUN rustup target add armv7-linux-androideabi
RUN rustup target add i686-linux-android
RUN rustup target add x86_64-linux-android
# WASM
RUN rustup target add wasm32-unknown-unknown
# Caching tool
RUN cargo install cargo-chef
# Install Linux cross-platform tooling
RUN curl -O https://ziglang.org/builds/zig-linux-x86_64-$ZIG_VERSION.tar.xz
RUN tar -C /usr/local -xJf zig-linux-x86_64-$ZIG_VERSION.tar.xz
RUN mv /usr/local/zig-linux-x86_64-$ZIG_VERSION /usr/local/zig
RUN cargo install cargo-zigbuild
SAVE ARTIFACT $RUSTUP_HOME rustup
SAVE ARTIFACT $CARGO_HOME cargo
SAVE ARTIFACT /usr/local/cargo/bin/cargo-zigbuild
SAVE ARTIFACT /usr/local/zig
# Install android tooling
deps-android:
FROM +deps-base
BUILD +deps-protoc
COPY +deps-protoc/bin/* /usr/local/bin/
BUILD +deps-capnp
COPY +deps-capnp/bin/* /usr/local/bin/
COPY +deps-capnp/lib/* /usr/local/lib/
BUILD +deps-rust
COPY +deps-rust/cargo /usr/local/cargo
COPY +deps-rust/rustup /usr/local/rustup
COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild
COPY +deps-rust/zig /usr/local/zig
RUN apt-get install -y openjdk-9-jdk-headless
RUN mkdir /Android; mkdir /Android/Sdk
RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-9123335_latest.zip
RUN cd /Android; unzip /Android/cmdline-tools.zip
RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33 cmdline-tools\;latest
RUN rm -rf /Android/cmdline-tools
RUN apt-get clean
# Just linux build not android
deps-linux:
FROM +deps-base
BUILD +deps-protoc
COPY +deps-protoc/bin/* /usr/local/bin/
BUILD +deps-capnp
COPY +deps-capnp/bin/* /usr/local/bin/
COPY +deps-capnp/lib/* /usr/local/lib/
BUILD +deps-rust
COPY +deps-rust/cargo /usr/local/cargo
COPY +deps-rust/rustup /usr/local/rustup
COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild
COPY +deps-rust/zig /usr/local/zig
build-linux-cache:
FROM +deps-linux
RUN mkdir veilid-cli veilid-core veilid-server veilid-tools veilid-wasm veilid-flutter veilid-flutter/rust
COPY --dir .cargo scripts Cargo.lock Cargo.toml .
COPY veilid-cli/Cargo.toml veilid-cli
COPY veilid-core/Cargo.toml veilid-core
COPY veilid-server/Cargo.toml veilid-server
COPY veilid-tools/Cargo.toml veilid-tools
COPY veilid-flutter/rust/Cargo.lock veilid-flutter/rust/Cargo.toml veilid-flutter/rust
COPY veilid-wasm/Cargo.toml veilid-wasm
RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> .cargo/config.toml
RUN cargo chef prepare --recipe-path recipe.json
RUN cargo chef cook --recipe-path recipe.json
RUN echo $PROJECT_PATH
SAVE ARTIFACT target
ARG CI_PROJECT_PATH=veilid/veilid
SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest
code-linux:
# This target will either use the full earthly cache of local use (+build-linux-cache), or will use a containerized
# version of the +build-linux-cache from the registry
ARG BASE=local
IF [ "$BASE" = "local" ]
FROM +build-linux-cache
ELSE
ARG CI_PROJECT_PATH=veilid/veilid
FROM registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest
# FROM registry.gitlab.com/veilid/build-cache:latest
END
COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
# Code + Linux + Android deps
code-android:
FROM +deps-android
COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid
RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml
RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml
# Clippy only
clippy:
FROM +code-linux
RUN cargo clippy
# Build
build-release:
FROM +code-linux
RUN cargo build --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
SAVE ARTIFACT ./target/release AS LOCAL ./target/release
build:
FROM +code-linux
RUN cargo build -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
SAVE ARTIFACT ./target/debug AS LOCAL ./target/debug
build-linux-amd64:
FROM +code-linux
RUN cargo zigbuild --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
build-linux-amd64-debug:
FROM +code-linux
RUN cargo zigbuild --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu
build-linux-arm64:
FROM +code-linux
RUN cargo zigbuild --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
SAVE ARTIFACT ./target/aarch64-unknown-linux-gnu AS LOCAL ./target/artifacts/aarch64-unknown-linux-gnu
build-android:
FROM +code-android
WORKDIR /veilid/veilid-core
ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/
RUN cargo build --target aarch64-linux-android --release
RUN cargo build --target armv7-linux-androideabi --release
RUN cargo build --target i686-linux-android --release
RUN cargo build --target x86_64-linux-android --release
WORKDIR /veilid
SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android
SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi
SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android
SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android
# Unit tests
unit-tests-linux:
FROM +code-linux
ENV RUST_BACKTRACE=1
RUN cargo test -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
# TODO: Change t0 cross so that they work on any platform
unit-tests-linux-amd64:
FROM +code-linux
ENV RUST_BACKTRACE=1
RUN cargo test --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
unit-tests-linux-arm64:
FROM +code-linux
ENV RUST_BACKTRACE=1
RUN cargo test --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core
# Package
package-linux-amd64-deb:
FROM +build-linux-amd64
#################################
### DEBIAN DPKG .DEB FILES
#################################
COPY --dir package /veilid
# veilid-server
RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh amd64 x86_64-unknown-linux-gnu
SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/
# veilid-cli
RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh amd64 x86_64-unknown-linux-gnu
# save artifacts
SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/
package-linux-amd64-rpm:
FROM --platform amd64 rockylinux:8
RUN yum install -y createrepo rpm-build rpm-sign yum-utils rpmdevtools
RUN rpmdev-setuptree
#################################
### RPMBUILD .RPM FILES
#################################
RUN mkdir -p /veilid/target
COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml package /veilid
COPY +build-linux-amd64/x86_64-unknown-linux-gnu /veilid/target/x86_64-unknown-linux-gnu
RUN mkdir -p /rpm-work-dir/veilid-server
# veilid-server
RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh x86_64 x86_64-unknown-linux-gnu
#SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/x86_64/*.rpm AS LOCAL ./target/packages/
# veilid-cli
RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh x86_64 x86_64-unknown-linux-gnu
# save artifacts
SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/x86_64/*.rpm AS LOCAL ./target/packages/
package-linux-arm64-deb:
FROM +build-linux-arm64
#################################
### DEBIAN DPKG .DEB FILES
#################################
COPY --dir package /veilid
# veilid-server
RUN /veilid/package/debian/earthly_make_veilid_server_deb.sh arm64 aarch64-unknown-linux-gnu
SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/
# veilid-cli
RUN /veilid/package/debian/earthly_make_veilid_cli_deb.sh arm64 aarch64-unknown-linux-gnu
# save artifacts
SAVE ARTIFACT --keep-ts /dpkg/out/*.deb AS LOCAL ./target/packages/
package-linux-arm64-rpm:
FROM --platform arm64 rockylinux:8
RUN yum install -y createrepo rpm-build rpm-sign yum-utils rpmdevtools
RUN rpmdev-setuptree
#################################
### RPMBUILD .RPM FILES
#################################
RUN mkdir -p /veilid/target
COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml package /veilid
COPY +build-linux-arm64/aarch64-unknown-linux-gnu /veilid/target/aarch64-unknown-linux-gnu
RUN mkdir -p /rpm-work-dir/veilid-server
# veilid-server
RUN veilid/package/rpm/veilid-server/earthly_make_veilid_server_rpm.sh aarch64 aarch64-unknown-linux-gnu
#SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/
# veilid-cli
RUN veilid/package/rpm/veilid-cli/earthly_make_veilid_cli_rpm.sh aarch64 aarch64-unknown-linux-gnu
# save artifacts
SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/
package-linux-amd64:
BUILD +package-linux-amd64-deb
BUILD +package-linux-amd64-rpm
package-linux-arm64:
BUILD +package-linux-arm64-deb
BUILD +package-linux-arm64-rpm
package-linux:
BUILD +package-linux-amd64
BUILD +package-linux-arm64
Reference in New Issue View Git Blame Copy Permalink