diff --git a/Earthfile b/Earthfile index 53af93f5..a7d173cc 100644 --- a/Earthfile +++ b/Earthfile @@ -26,22 +26,6 @@ deps-base: RUN apt-get -y update RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev -# Install Cap'n Proto -deps-capnp: - FROM +deps-base - COPY scripts/earthly/install_capnproto.sh / - RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh - SAVE ARTIFACT /usr/local/bin/capnp* bin/ - SAVE ARTIFACT /usr/local/lib/lib* lib/ - SAVE ARTIFACT /usr/local/lib/pkgconfig lib/ - -# Install protoc -deps-protoc: - FROM +deps-base - COPY scripts/earthly/install_protoc.sh / - RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh - SAVE ARTIFACT /usr/local/bin/protoc bin/protoc - # Install Rust deps-rust: FROM +deps-base @@ -75,11 +59,6 @@ deps-rust: # Install android tooling deps-android: FROM +deps-base - BUILD +deps-protoc - COPY +deps-protoc/bin/* /usr/local/bin/ - BUILD +deps-capnp - COPY +deps-capnp/bin/* /usr/local/bin/ - COPY +deps-capnp/lib/* /usr/local/lib/ BUILD +deps-rust COPY +deps-rust/cargo /usr/local/cargo COPY +deps-rust/rustup /usr/local/rustup @@ -96,11 +75,6 @@ deps-android: # Just linux build not android deps-linux: FROM +deps-base - BUILD +deps-protoc - COPY +deps-protoc/bin/* /usr/local/bin/ - BUILD +deps-capnp - COPY +deps-capnp/bin/* /usr/local/bin/ - COPY +deps-capnp/lib/* /usr/local/lib/ BUILD +deps-rust COPY +deps-rust/cargo /usr/local/cargo COPY +deps-rust/rustup /usr/local/rustup diff --git a/dev-setup/setup_linux.sh b/dev-setup/setup_linux.sh index 019169de..107acb0b 100755 --- a/dev-setup/setup_linux.sh +++ b/dev-setup/setup_linux.sh @@ -17,75 +17,75 @@ if ! lsb_release -d | grep -qEi 'debian|buntu|mint' && [ -z "$(command -v dnf)" echo Not a supported Linux exit 1 fi + while true; do -read -p "Did you install Android SDK? Y/N " response + read -p "Did you install Android SDK? Y/N " response -case $response in - [yY] ) echo Checking android setup...; + case $response in + [yY] ) echo Checking android setup...; -# ensure ANDROID_SDK_ROOT is defined and exists -if [ -d "$ANDROID_SDK_ROOT" ]; then - echo '[X] $ANDROID_SDK_ROOT is defined and exists' -else - echo '$ANDROID_SDK_ROOT is not defined or does not exist' - exit 1 -fi + # ensure ANDROID_SDK_ROOT is defined and exists + if [ -d "$ANDROID_SDK_ROOT" ]; then + echo '[X] $ANDROID_SDK_ROOT is defined and exists' + else + echo '$ANDROID_SDK_ROOT is not defined or does not exist' + exit 1 + fi -# ensure Android Command Line Tools exist -if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then - echo '[X] Android command line tools are installed' -else - echo 'Android command line tools are not installed' - exit 1 -fi + # ensure Android Command Line Tools exist + if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then + echo '[X] Android command line tools are installed' + else + echo 'Android command line tools are not installed' + exit 1 + fi -# ensure ANDROID_NDK_HOME is defined and exists -if [ -d "$ANDROID_NDK_HOME" ]; then - echo '[X] $ANDROID_NDK_HOME is defined and exists' -else - echo '$ANDROID_NDK_HOME is not defined or does not exist' - exit 1 -fi + # ensure ANDROID_NDK_HOME is defined and exists + if [ -d "$ANDROID_NDK_HOME" ]; then + echo '[X] $ANDROID_NDK_HOME is defined and exists' + else + echo '$ANDROID_NDK_HOME is not defined or does not exist' + exit 1 + fi -# ensure ndk is installed -if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then - echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME' -else - echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME' - exit 1 -fi + # ensure ndk is installed + if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then + echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME' + else + echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME' + exit 1 + fi -# ensure cmake is installed -if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then - echo '[X] Android SDK CMake is installed' -else - echo 'Android SDK CMake is not installed' - exit 1 -fi + # ensure cmake is installed + if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then + echo '[X] Android SDK CMake is installed' + else + echo 'Android SDK CMake is not installed' + exit 1 + fi -# ensure emulator is installed -if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then - echo '[X] Android SDK emulator is installed' -else - echo 'Android SDK emulator is not installed' - exit 1 -fi + # ensure emulator is installed + if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then + echo '[X] Android SDK emulator is installed' + else + echo 'Android SDK emulator is not installed' + exit 1 + fi -# ensure adb is installed -if command -v adb &> /dev/null; then - echo '[X] adb is available in the path' -else - echo 'adb is not available in the path' - exit 1 -fi -break;; -[nN] ) echo Skipping android SDK config check...; -break;; - -* ) echo invalid response;; - -esac + # ensure adb is installed + if command -v adb &> /dev/null; then + echo '[X] adb is available in the path' + else + echo 'adb is not available in the path' + exit 1 + fi + break;; + [nN] ) echo Skipping Android SDK config check...; + break;; + * ) echo invalid response;; + esac done + # ensure rustup is installed if command -v rustup &> /dev/null; then echo '[X] rustup is available in the path' @@ -119,7 +119,21 @@ cargo install wasm-bindgen-cli wasm-pack cargo-edit # install pip packages pip3 install --upgrade bumpversion -# Install capnproto using the same mechanism as our earthly build -$SCRIPTDIR/../scripts/earthly/install_capnproto.sh -# Install protoc using the same mechanism as our earthly build -$SCRIPTDIR/../scripts/earthly/install_protoc.sh +# install capnp +while true; do + read -p "Will you be modifying the capnproto schema? Y/N (say N if unsure)" response + + case $response in + [yY] ) echo Installing capnproto...; + + # Install capnproto using the same mechanism as our earthly build + $SCRIPTDIR/../scripts/earthly/install_capnproto.sh + + break;; + [nN] ) echo Skipping capnproto installation...; + break;; + * ) echo invalid response;; + esac +done + + diff --git a/dev-setup/setup_macos.sh b/dev-setup/setup_macos.sh index 0ec96c1c..34ed33fa 100755 --- a/dev-setup/setup_macos.sh +++ b/dev-setup/setup_macos.sh @@ -7,69 +7,77 @@ if [ ! "$(uname)" == "Darwin" ]; then echo Not running on MacOS exit 1 fi -read -p "Did you install Android SDK? Y/N " response -case $response in - [yY] ) echo Checking android setup...; -# ensure ANDROID_SDK_ROOT is defined and exists -if [ -d "$ANDROID_SDK_ROOT" ]; then - echo '[X] $ANDROID_SDK_ROOT is defined and exists' -else - echo '$ANDROID_SDK_ROOT is not defined or does not exist' - exit 1 -fi +while true; do -# ensure Android Command Line Tools exist -if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then - echo '[X] Android command line tools are installed' -else - echo 'Android command line tools are not installed' - exit 1 -fi + read -p "Did you install Android SDK? Y/N " response + case $response in + [yY] ) echo Checking android setup...; + # ensure ANDROID_SDK_ROOT is defined and exists + if [ -d "$ANDROID_SDK_ROOT" ]; then + echo '[X] $ANDROID_SDK_ROOT is defined and exists' + else + echo '$ANDROID_SDK_ROOT is not defined or does not exist' + exit 1 + fi -# ensure Android SDK packages are installed -$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33 + # ensure Android Command Line Tools exist + if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then + echo '[X] Android command line tools are installed' + else + echo 'Android command line tools are not installed' + exit 1 + fi -# ensure ANDROID_NDK_HOME is defined and exists -if [ -d "$ANDROID_NDK_HOME" ]; then - echo '[X] $ANDROID_NDK_HOME is defined and exists' -else - echo '$ANDROID_NDK_HOME is not defined or does not exist' - exit 1 -fi + # ensure Android SDK packages are installed + $ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33 -# ensure ndk is installed -if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then - echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME' -else - echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME' - exit 1 -fi + # ensure ANDROID_NDK_HOME is defined and exists + if [ -d "$ANDROID_NDK_HOME" ]; then + echo '[X] $ANDROID_NDK_HOME is defined and exists' + else + echo '$ANDROID_NDK_HOME is not defined or does not exist' + exit 1 + fi -# ensure cmake is installed -if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then - echo '[X] Android SDK CMake is installed' -else - echo 'Android SDK CMake is not installed' - exit 1 -fi + # ensure ndk is installed + if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then + echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME' + else + echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME' + exit 1 + fi -# ensure emulator is installed -if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then - echo '[X] Android SDK emulator is installed' -else - echo 'Android SDK emulator is not installed' - exit 1 -fi + # ensure cmake is installed + if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then + echo '[X] Android SDK CMake is installed' + else + echo 'Android SDK CMake is not installed' + exit 1 + fi -# ensure adb is installed -if command -v adb &> /dev/null; then - echo '[X] adb is available in the path' -else - echo 'adb is not available in the path' - exit 1 -fi -esac + # ensure emulator is installed + if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then + echo '[X] Android SDK emulator is installed' + else + echo 'Android SDK emulator is not installed' + exit 1 + fi + + # ensure adb is installed + if command -v adb &> /dev/null; then + echo '[X] adb is available in the path' + else + echo 'adb is not available in the path' + exit 1 + fi + break;; + [nN] ) echo Skipping Android SDK config check...; + break;; + + * ) echo invalid response;; + esac +done # ensure brew is installed if command -v brew &> /dev/null; then @@ -130,7 +138,7 @@ if [ "$BREW_USER" == "" ]; then BREW_USER=`whoami` fi fi -sudo -H -u $BREW_USER brew install capnp cmake wabt llvm protobuf openjdk@17 jq +sudo -H -u $BREW_USER brew install capnp cmake wabt llvm openjdk@17 jq # install targets rustup target add aarch64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-darwin x86_64-apple-ios wasm32-unknown-unknown aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android diff --git a/scripts/earthly/install_protoc.sh b/scripts/earthly/install_protoc.sh deleted file mode 100755 index d01a780e..00000000 --- a/scripts/earthly/install_protoc.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" -PROTOC_VERSION="24.3" # Keep in sync with veilid-core/build.rs - -UNAME_M=$(uname -m) -if [[ "$UNAME_M" == "x86_64" ]]; then - PROTOC_ARCH=x86_64 -elif [[ "$UNAME_M" == "aarch64" ]]; then - PROTOC_ARCH=aarch_64 -else - echo Unsupported build architecture - exit 1 -fi - -mkdir /tmp/protoc-install -pushd /tmp/protoc-install -curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-linux-$PROTOC_ARCH.zip -unzip protoc-$PROTOC_VERSION-linux-$PROTOC_ARCH.zip -if [ "$EUID" -ne 0 ]; then - if command -v checkinstall &> /dev/null; then - sudo checkinstall --pkgversion=$PROTOC_VERSION -y cp -r bin include /usr/local/ - cp *.deb ~ - else - sudo cp -r bin include /usr/local/ - fi - popd - sudo rm -rf /tmp/protoc-install -else - if command -v checkinstall &> /dev/null; then - checkinstall --pkgversion=$PROTOC_VERSION -y cp -r bin include /usr/local/ - cp *.deb ~ - else - cp -r bin include /usr/local/ - fi - popd - rm -rf /tmp/protoc-install -fi diff --git a/veilid-core/build.rs b/veilid-core/build.rs index a40f249a..226e9d48 100644 --- a/veilid-core/build.rs +++ b/veilid-core/build.rs @@ -1,20 +1,16 @@ use glob::glob; use std::{ - env, + env, fs, io, + path::Path, process::{Command, Stdio}, }; -const CAPNP_VERSION: &str = "1.0.1"; // Keep in sync with scripts/install_capnp.sh -const PROTOC_VERSION: &str = "24.3"; // Keep in sync with scripts/install_protoc.sh +const CAPNP_VERSION: &str = "1.0.1"; fn get_desired_capnp_version_string() -> String { CAPNP_VERSION.to_string() } -fn get_desired_protoc_version_string() -> String { - PROTOC_VERSION.to_string() -} - fn get_capnp_version_string() -> String { let output = Command::new("capnp") .arg("--version") @@ -32,35 +28,29 @@ fn get_capnp_version_string() -> String { s[20..].to_owned() } -fn get_protoc_version_string() -> String { - let output = Command::new("protoc") - .arg("--version") - .stdout(Stdio::piped()) - .output() - .expect("protoc was not in the PATH"); - let s = String::from_utf8(output.stdout) - .expect("'protoc --version' output was not a valid string") - .trim() - .to_owned(); +fn is_input_file_outdated(input: P1, output: P2) -> io::Result +where + P1: AsRef, + P2: AsRef, +{ + let out_meta = fs::metadata(output); + if let Ok(meta) = out_meta { + let output_mtime = meta.modified()?; - if !s.starts_with("libprotoc ") { - panic!("invalid protoc version string: {}", s); + // if input file is more recent than our output, we are outdated + let input_meta = fs::metadata(input)?; + let input_mtime = input_meta.modified()?; + + Ok(input_mtime > output_mtime) + } else { + // output file not found, we are outdated + Ok(true) } - s[10..].to_owned() } -fn main() { - if std::env::var("DOCS_RS").is_ok() - || std::env::var("CARGO_CFG_DOC").is_ok() - || std::env::var("BUILD_DOCS").is_ok() - { - return; - } - +fn do_capnp_build() { let desired_capnp_version_string = get_desired_capnp_version_string(); let capnp_version_string = get_capnp_version_string(); - let desired_protoc_version_string = get_desired_protoc_version_string(); - let protoc_version_string = get_protoc_version_string(); // Check capnp version let desired_capnp_major_version = desired_capnp_version_string @@ -89,45 +79,16 @@ fn main() { ); } - // Check protoc version - let desired_protoc_major_version = desired_protoc_version_string - .split_once('.') - .unwrap() - .0 - .parse::() - .expect("should be valid int"); - if protoc_version_string - .split_once('.') - .unwrap() - .0 - .parse::() - .expect("should be valid int") - < desired_protoc_major_version - { - panic!( - "protoc version should be at least major version {} but is {}", - desired_protoc_major_version, protoc_version_string - ); - } else if protoc_version_string != desired_protoc_version_string { - println!( - "cargo:warning=protoc version may be untested: {}", - protoc_version_string - ); - } - ::capnpc::CompilerCommand::new() .file("proto/veilid.capnp") .output_path(".") .run() .expect("compiling schema"); +} - // Fix for missing __extenddftf2 on Android x86_64 Emulator +// Fix for missing __extenddftf2 on Android x86_64 Emulator +fn fix_android_emulator() { let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); - // if target_os == "android" || target_os == "linux" { - // println!("cargo:rustc-link-lib=stdc++"); - // } else { - // println!("cargo:rustc-link-lib=c++"); - // } let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap(); if target_arch == "x86_64" && target_os == "android" { let missing_library = "clang_rt.builtins-x86_64-android"; @@ -142,3 +103,18 @@ fn main() { println!("cargo:rustc-link-lib=static={missing_library}"); } } + +fn main() { + if std::env::var("DOCS_RS").is_ok() + || std::env::var("CARGO_CFG_DOC").is_ok() + || std::env::var("BUILD_DOCS").is_ok() + { + return; + } + + if is_input_file_outdated("./proto/veilid.capnp", "./proto/veilid_capnp.rs").unwrap() { + do_capnp_build(); + } + + fix_android_emulator(); +}