diff --git a/.gitignore b/.gitignore index 4d82ead2..b410b5be 100644 --- a/.gitignore +++ b/.gitignore @@ -236,4 +236,7 @@ veilid-python/demo/.demokeys .vscode/ ## Ignore emacs backup files -~* \ No newline at end of file +~* + +# Earthly temporary build output +.tmp-earthly-out/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b4c975c4..6e132a10 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,29 +5,53 @@ variables: GIT_SUBMODULE_STRATEGY: normal stages: + - prepare - test - build_packages - release - distribute -unit_test_linux: - stage: test +# base earthly setup for jobs +.base: + tags: [ saas-linux-medium-amd64 ] image: docker services: - docker:dind + +.earthly: &earthly_setup + - apk update && apk add git + - wget https://github.com/earthly/earthly/releases/download/v0.7.15/earthly-linux-amd64 -O /usr/local/bin/earthly + - chmod +x /usr/local/bin/earthly + - earthly bootstrap + - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY + - test "$CI_PROJECT_PATH" != "veilid/veilid" && project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH" + +# Create the build container if: +# - no container in the registry +# - run as part of a schedule +# - run manually from the pipelines web page +build_cache: + extends: .base + stage: prepare + script: + - if ! docker manifest inspect registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest > /dev/null || [[ "$CI_PIPELINE_SOURCE" == "schedule" ]] || [[ "$CI_PIPELINE_SOURCE" == "web" ]] ; then + - *earthly_setup + - earthly --use-inline-cache --save-inline-cache --strict --push -P +build-linux-cache $project_args + - else + - echo "No need to rebuild" + - fi + +# Runs a basic unit test build, this task will use the `build-cache:latest` as set up in the projects Container Registry +test_build: + extends: .base + stage: test only: - main + - pushes - merge_requests - before_script: - - apk update && apk add git - - wget https://github.com/earthly/earthly/releases/download/v0.6.2/earthly-linux-amd64 -O /usr/local/bin/earthly - - chmod +x /usr/local/bin/earthly - - earthly bootstrap - - echo $CI_REGISTRY_PASSWORD | docker login -u $CI_REGISTRY_USER --password-stdin $CI_REGISTRY script: - - if [ "$CI_COMMIT_BRANCH" == "main" ]; then cache_args="--save-inline-cache --push"; fi - - if [ "$CI_PROJECT_PATH" != "veilid/veilid" ]; then project_args="--CI_PROJECT_PATH=$CI_PROJECT_PATH"; fi - - earthly --use-inline-cache $cache_args --ci +unit-tests-linux $project_args + - *earthly_setup + - earthly --use-inline-cache +unit-tests-linux --BASE=container $project_args resource_group: test when: manual diff --git a/Earthfile b/Earthfile index 7a2d2e6b..53af93f5 100644 --- a/Earthfile +++ b/Earthfile @@ -1,10 +1,27 @@ -VERSION 0.6 +VERSION 0.7 + +######################################################################################################################## +## ARGUMENTS +## +## CI_PROJECT_PATH - used so that forks can refer to themselves, e.g. to use the fork's own registry cache in the +## `+build-linux-cache` target, and defaulting to `veilid/veilid` if not specified +## +## BASE - tells the build whether it should run in the default mode which runs the complete build, or run by starting +## with the remote `container` value which uses `build-cache:latest` as set up in the projects Container Registry +## +######################################################################################################################## # Start with older Ubuntu to ensure GLIBC symbol versioning support for older linux # Ensure we are using an amd64 platform because some of these targets use cross-platform tooling FROM ubuntu:18.04 +ENV ZIG_VERSION=0.11.0-dev.3978+711b4e93e +ENV RUSTUP_HOME=/usr/local/rustup +ENV CARGO_HOME=/usr/local/cargo +ENV PATH=$PATH:/usr/local/cargo/bin:/usr/local/zig +ENV LD_LIBRARY_PATH=/usr/local/lib +WORKDIR /veilid -# Install build prerequisites +# Install build prerequisites & setup required directories deps-base: RUN apt-get -y update RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev @@ -14,25 +31,27 @@ deps-capnp: FROM +deps-base COPY scripts/earthly/install_capnproto.sh / RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh + SAVE ARTIFACT /usr/local/bin/capnp* bin/ + SAVE ARTIFACT /usr/local/lib/lib* lib/ + SAVE ARTIFACT /usr/local/lib/pkgconfig lib/ # Install protoc deps-protoc: - FROM +deps-capnp + FROM +deps-base COPY scripts/earthly/install_protoc.sh / RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh + SAVE ARTIFACT /usr/local/bin/protoc bin/protoc # Install Rust deps-rust: - FROM +deps-protoc - ENV RUSTUP_HOME=/usr/local/rustup - ENV CARGO_HOME=/usr/local/cargo - ENV PATH=/usr/local/cargo/bin:$PATH - RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal + FROM +deps-base + RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -c clippy --no-modify-path --profile minimal RUN chmod -R a+w $RUSTUP_HOME $CARGO_HOME; \ rustup --version; \ cargo --version; \ rustc --version; - # ARM64 Linux + # Linux + RUN rustup target add x86_64-unknown-linux-gnu RUN rustup target add aarch64-unknown-linux-gnu # Android RUN rustup target add aarch64-linux-android @@ -41,20 +60,31 @@ deps-rust: RUN rustup target add x86_64-linux-android # WASM RUN rustup target add wasm32-unknown-unknown - -# Install Linux cross-platform tooling -deps-cross: - FROM +deps-rust - RUN curl https://ziglang.org/builds/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e.tar.xz | tar -C /usr/local -xJf - - RUN mv /usr/local/zig-linux-x86_64-0.11.0-dev.3978+711b4e93e /usr/local/zig - ENV PATH=$PATH:/usr/local/zig + # Caching tool + RUN cargo install cargo-chef + # Install Linux cross-platform tooling + RUN curl -O https://ziglang.org/builds/zig-linux-x86_64-$ZIG_VERSION.tar.xz + RUN tar -C /usr/local -xJf zig-linux-x86_64-$ZIG_VERSION.tar.xz + RUN mv /usr/local/zig-linux-x86_64-$ZIG_VERSION /usr/local/zig RUN cargo install cargo-zigbuild - RUN rustup target add x86_64-unknown-linux-gnu - RUN rustup target add aarch64-unknown-linux-gnu + SAVE ARTIFACT $RUSTUP_HOME rustup + SAVE ARTIFACT $CARGO_HOME cargo + SAVE ARTIFACT /usr/local/cargo/bin/cargo-zigbuild + SAVE ARTIFACT /usr/local/zig # Install android tooling deps-android: - FROM +deps-cross + FROM +deps-base + BUILD +deps-protoc + COPY +deps-protoc/bin/* /usr/local/bin/ + BUILD +deps-capnp + COPY +deps-capnp/bin/* /usr/local/bin/ + COPY +deps-capnp/lib/* /usr/local/lib/ + BUILD +deps-rust + COPY +deps-rust/cargo /usr/local/cargo + COPY +deps-rust/rustup /usr/local/rustup + COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild + COPY +deps-rust/zig /usr/local/zig RUN apt-get install -y openjdk-9-jdk-headless RUN mkdir /Android; mkdir /Android/Sdk RUN curl -o /Android/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-9123335_latest.zip @@ -62,18 +92,51 @@ deps-android: RUN yes | /Android/cmdline-tools/bin/sdkmanager --sdk_root=/Android/Sdk build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33 cmdline-tools\;latest RUN rm -rf /Android/cmdline-tools RUN apt-get clean - + # Just linux build not android deps-linux: - FROM +deps-cross - RUN apt-get clean + FROM +deps-base + BUILD +deps-protoc + COPY +deps-protoc/bin/* /usr/local/bin/ + BUILD +deps-capnp + COPY +deps-capnp/bin/* /usr/local/bin/ + COPY +deps-capnp/lib/* /usr/local/lib/ + BUILD +deps-rust + COPY +deps-rust/cargo /usr/local/cargo + COPY +deps-rust/rustup /usr/local/rustup + COPY +deps-rust/cargo-zigbuild /usr/local/cargo/bin/cargo-zigbuild + COPY +deps-rust/zig /usr/local/zig -# Code + Linux deps -code-linux: +build-linux-cache: FROM +deps-linux + RUN mkdir veilid-cli veilid-core veilid-server veilid-tools veilid-wasm veilid-flutter veilid-flutter/rust + COPY --dir .cargo scripts Cargo.lock Cargo.toml . + COPY veilid-cli/Cargo.toml veilid-cli + COPY veilid-core/Cargo.toml veilid-core + COPY veilid-server/Cargo.toml veilid-server + COPY veilid-tools/Cargo.toml veilid-tools + COPY veilid-flutter/rust/Cargo.lock veilid-flutter/rust/Cargo.toml veilid-flutter/rust + COPY veilid-wasm/Cargo.toml veilid-wasm + RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> .cargo/config.toml + RUN cargo chef prepare --recipe-path recipe.json + RUN cargo chef cook --recipe-path recipe.json + RUN echo $PROJECT_PATH + SAVE ARTIFACT target + ARG CI_PROJECT_PATH=veilid/veilid + SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest + +code-linux: + # This target will either use the full earthly cache of local use (+build-linux-cache), or will use a containerized + # version of the +build-linux-cache from the registry + ARG BASE=local + IF [ "$BASE" = "local" ] + FROM +build-linux-cache + ELSE + ARG CI_PROJECT_PATH=veilid/veilid + FROM registry.gitlab.com/$CI_PROJECT_PATH/build-cache:latest + # FROM registry.gitlab.com/veilid/build-cache:latest + END COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid - RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml - WORKDIR /veilid # Code + Linux + Android deps code-android: @@ -81,89 +144,68 @@ code-android: COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml - WORKDIR /veilid # Clippy only clippy: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid RUN cargo clippy - SAVE IMAGE --push registry.gitl ab.com/$CI_PROJECT_PATH/clippy:latest # Build build-release: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid RUN cargo build --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE ARTIFACT ./target/release AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-release:latest + SAVE ARTIFACT ./target/release AS LOCAL ./target/release build: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid RUN cargo build -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE ARTIFACT ./target/debug AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build:latest + SAVE ARTIFACT ./target/debug AS LOCAL ./target/debug build-linux-amd64: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid - RUN cargo build --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core + RUN cargo zigbuild --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64:latest build-linux-amd64-debug: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid - RUN cargo build --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core + RUN cargo zigbuild --target x86_64-unknown-linux-gnu -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-amd64-debug:latest build-linux-arm64: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid RUN cargo zigbuild --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core SAVE ARTIFACT ./target/aarch64-unknown-linux-gnu AS LOCAL ./target/artifacts/aarch64-unknown-linux-gnu - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-linux-arm64:latest -# build-android: -# FROM +code-android -# ARG CI_PROJECT_PATH=veilid/veilid -# WORKDIR /veilid/veilid-core -# ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/ -# RUN cargo build --target aarch64-linux-android --release -# RUN cargo build --target armv7-linux-androideabi --release -# RUN cargo build --target i686-linux-android --release -# RUN cargo build --target x86_64-linux-android --release -# WORKDIR /veilid -# SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android -# SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi -# SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android -# SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android -# SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/build-android:latest +build-android: + FROM +code-android + WORKDIR /veilid/veilid-core + ENV PATH=$PATH:/Android/Sdk/ndk/25.1.8937393/toolchains/llvm/prebuilt/linux-x86_64/bin/ + RUN cargo build --target aarch64-linux-android --release + RUN cargo build --target armv7-linux-androideabi --release + RUN cargo build --target i686-linux-android --release + RUN cargo build --target x86_64-linux-android --release + WORKDIR /veilid + SAVE ARTIFACT ./target/aarch64-linux-android AS LOCAL ./target/artifacts/aarch64-linux-android + SAVE ARTIFACT ./target/armv7-linux-androideabi AS LOCAL ./target/artifacts/armv7-linux-androideabi + SAVE ARTIFACT ./target/i686-linux-android AS LOCAL ./target/artifacts/i686-linux-android + SAVE ARTIFACT ./target/x86_64-linux-android AS LOCAL ./target/artifacts/x86_64-linux-android # Unit tests unit-tests-linux: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid ENV RUST_BACKTRACE=1 - RUN cargo test --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux:latest + RUN cargo test -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core # TODO: Change t0 cross so that they work on any platform unit-tests-linux-amd64: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid ENV RUST_BACKTRACE=1 RUN cargo test --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-amd64:latest unit-tests-linux-arm64: FROM +code-linux - ARG CI_PROJECT_PATH=veilid/veilid ENV RUST_BACKTRACE=1 RUN cargo test --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core - SAVE IMAGE --push registry.gitlab.com/$CI_PROJECT_PATH/unit-tests-linux-arm64:latest # Package package-linux-amd64-deb: @@ -232,8 +274,6 @@ package-linux-arm64-rpm: # save artifacts SAVE ARTIFACT --keep-ts /root/rpmbuild/RPMS/aarch64/*.rpm AS LOCAL ./target/packages/ - - package-linux-amd64: BUILD +package-linux-amd64-deb BUILD +package-linux-amd64-rpm diff --git a/veilid-cli/Cargo.toml b/veilid-cli/Cargo.toml index ee824b44..92418e7b 100644 --- a/veilid-cli/Cargo.toml +++ b/veilid-cli/Cargo.toml @@ -6,6 +6,7 @@ version = "0.2.3" authors = ["Veilid Team "] edition = "2021" license = "MPL-2.0" +resolver = "2" [[bin]] name = "veilid-cli" diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 783d4ead..f32a2b2c 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -8,9 +8,11 @@ authors = ["Veilid Team "] edition = "2021" build = "build.rs" license = "MPL-2.0" +resolver = "2" [lib] crate-type = ["cdylib", "staticlib", "rlib"] +path = "src/lib.rs" [features] diff --git a/veilid-flutter/rust/Cargo.toml b/veilid-flutter/rust/Cargo.toml index 11fb0f05..a7accaa5 100644 --- a/veilid-flutter/rust/Cargo.toml +++ b/veilid-flutter/rust/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" [lib] crate-type = ["cdylib", "staticlib", "rlib"] +path = "src/lib.rs" [features] default = ["rt-tokio", "veilid-core/default"] diff --git a/veilid-server/Cargo.toml b/veilid-server/Cargo.toml index cedb3834..cd1845b6 100644 --- a/veilid-server/Cargo.toml +++ b/veilid-server/Cargo.toml @@ -7,6 +7,7 @@ description = "Veilid Server" authors = ["Veilid Team "] license = "MPL-2.0" edition = "2021" +resolver = "2" [[bin]] name = "veilid-server" diff --git a/veilid-tools/Cargo.toml b/veilid-tools/Cargo.toml index ea8fd23e..cb049044 100644 --- a/veilid-tools/Cargo.toml +++ b/veilid-tools/Cargo.toml @@ -11,6 +11,7 @@ edition = "2021" [lib] # staticlib for iOS tests, cydlib for android tests, rlib for everything else crate-type = ["cdylib", "staticlib", "rlib"] +path = "src/lib.rs" [features] default = ["rt-tokio"] diff --git a/veilid-wasm/Cargo.toml b/veilid-wasm/Cargo.toml index 63d98e90..79292ba6 100644 --- a/veilid-wasm/Cargo.toml +++ b/veilid-wasm/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" [lib] crate-type = ["cdylib", "rlib"] +path = "src/lib.rs" [features] default = ["veilid-core/default-wasm"]