diff --git a/.bumpversion.toml b/.bumpversion.toml index babf00ec..d8adeaa4 100644 --- a/.bumpversion.toml +++ b/.bumpversion.toml @@ -46,6 +46,13 @@ version = "{current_version}"''' replace = '''name = "veilid-tools" version = "{new_version}"''' +[[tool.bumpversion.files]] +filename = "veilid-core-optimized/Cargo.toml" +search = '''name = "veilid-core-optimized" +version = "{current_version}"''' +replace = '''name = "veilid-core-optimized" +version = "{new_version}"''' + [[tool.bumpversion.files]] filename = "veilid-remote-api/Cargo.toml" search = '''name = "veilid-remote-api" diff --git a/.dagger/src/veilid/main.py b/.dagger/src/veilid/main.py index ec45b573..116f568f 100644 --- a/.dagger/src/veilid/main.py +++ b/.dagger/src/veilid/main.py @@ -212,6 +212,9 @@ class Veilid: ]).with_exec([ "sh", "-c", f"cargo binstall cargo-public-api --disable-strategies=compile --targets {DEFAULT_CARGO_MUSL_TARGET} || cargo install cargo-public-api --locked" + ]).with_exec([ + "sh", "-c", + f"cargo binstall cargo-sync-readme --disable-strategies=compile --targets {DEFAULT_CARGO_MUSL_TARGET} || cargo install cargo-sync-readme --locked" ]) # Install Zig for cross-compilation (no binstall available for zigbuild yet) @@ -271,7 +274,7 @@ class Veilid: "cargo", "chef", "cook", "--recipe-path", "recipe.json", "--profile=test", "--tests", "--target", DEFAULT_CARGO_TARGET, "--all-targets", "--locked", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", - "-p", "veilid-core", "-p", "veilid-remote-api" + "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" ]) # Cook dependencies for clippy x86_64-unknown-linux-gnu @@ -279,7 +282,7 @@ class Veilid: "cargo", "chef", "cook", "--recipe-path", "recipe.json", "--zigbuild=clippy", "--target", "x86_64-unknown-linux-gnu", "--all-targets", "--locked", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", - "-p", "veilid-core", "-p", "veilid-remote-api" + "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" ]) # Cook dependencies for clippy aarch64-unknown-linux-gnu @@ -287,7 +290,7 @@ class Veilid: "cargo", "chef", "cook", "--recipe-path", "recipe.json", "--zigbuild=clippy", "--target", "aarch64-unknown-linux-gnu", "--all-targets", "--locked", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", - "-p", "veilid-core", "-p", "veilid-remote-api" + "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" ]) # Cook dependencies for clippy x86_64-pc-windows-gnu @@ -295,7 +298,7 @@ class Veilid: "cargo", "chef", "cook", "--recipe-path", "recipe.json", "--zigbuild=clippy", "--target", "x86_64-pc-windows-gnu", "--all-targets", "--locked", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", - "-p", "veilid-core", "-p", "veilid-remote-api" + "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" ]) # Cook dependencies for clippy aarch64-apple-darwin @@ -303,7 +306,7 @@ class Veilid: "cargo", "chef", "cook", "--recipe-path", "recipe.json", "--zigbuild=clippy", "--target", "aarch64-apple-darwin", "--all-targets", "--locked", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", - "-p", "veilid-core", "-p", "veilid-remote-api" + "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" ]) # Cook WASM dependencies @@ -451,7 +454,7 @@ class Veilid: # which will screw up the build of veilid-server because it will automatically add that feature inappropriately out = ( container.with_exec([ - "cargo", "zigbuild", "--locked", "--target", target, "--release", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", "-p", "veilid-core", "-p", "veilid-remote-api" + "cargo", "zigbuild", "--locked", "--target", target, "--release", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" ]) .with_exec(["rm", "-rf", f"./target/{target}/release/.fingerprint"]) .with_exec(["rm", "-rf", f"./target/{target}/release/build"]) @@ -476,7 +479,7 @@ class Veilid: # which will screw up the build of veilid-server because it will automatically add that feature inappropriately out = ( container.with_exec([ - "cargo", "zigbuild", "--locked", "--target", target, "--release", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", "-p", "veilid-core", "-p", "veilid-remote-api" + "cargo", "zigbuild", "--locked", "--target", target, "--release", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" ]) .with_exec(["rm", "-rf", f"./target/{target}/release/.fingerprint"]) .with_exec(["rm", "-rf", f"./target/{target}/release/build"]) @@ -500,7 +503,7 @@ class Veilid: # which will screw up the build of veilid-server because it will automatically add that feature inappropriately out = ( container.with_exec([ - "cargo", "zigbuild", "--locked", "--target", target, "--release", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", "-p", "veilid-core", "-p", "veilid-remote-api" + "cargo", "zigbuild", "--locked", "--target", target, "--release", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" ]) .with_exec(["rm", "-rf", f"./target/{target}/release/.fingerprint"]) .with_exec(["rm", "-rf", f"./target/{target}/release/build"]) @@ -524,7 +527,7 @@ class Veilid: # # which will screw up the build of veilid-server because it will automatically add that feature inappropriately # out = ( # container.with_exec([ - # "cargo", "zigbuild", "--locked", "--target", target, "--release", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", "-p", "veilid-core", "-p", "veilid-remote-api" + # "cargo", "zigbuild", "--locked", "--target", target, "--release", "-p", "veilid-server", "-p", "veilid-cli", "-p", "veilid-tools", "-p", "veilid-core-optimized", "-p", "veilid-core", "-p", "veilid-remote-api" # ]) # .with_exec(["rm", "-rf", f"./target/{target}/release/.fingerprint"]) # .with_exec(["rm", "-rf", f"./target/{target}/release/build"]) @@ -551,6 +554,13 @@ class Veilid: result = await container.with_exec(["./scripts/_check_public_api.sh"]).combined_output() return f"Public API check completed:\n{result}" + @function + async def test_readme_sync(self, source: Annotated[dagger.Directory, Ignore(DIRECTORY_IGNORE_PATTERNS)], base: str = "local", ci_registry_image: str = VEILID_REPO, cache_tag: str = "latest") -> str: + """Check that every generated crate README matches its crate documentation""" + container = self.code_linux(source, base, ci_registry_image, cache_tag) + result = await container.with_exec(["./scripts/check_readme_sync.sh"]).combined_output() + return f"README sync check completed:\n{result}" + @function async def test_smoketest_all(self, source: Annotated[dagger.Directory, Ignore(DIRECTORY_IGNORE_PATTERNS)], base: str = "local", ci_registry_image: str = VEILID_REPO, cache_tag: str = "latest") -> str: """Run Smoketest checks for all published crates""" @@ -641,6 +651,7 @@ class Veilid: results = [] results.append(await self.test_msrv_all(source, base, ci_registry_image, cache_tag)) results.append(await self.test_public_api(source, base, ci_registry_image, cache_tag)) + results.append(await self.test_readme_sync(source, base, ci_registry_image, cache_tag)) results.append(await self.test_smoketest_all(source, base, ci_registry_image, cache_tag)) results.append(await self.test_check_wasm_builds_dart(source, base, ci_registry_image, cache_tag)) results.append(await self.test_check_wasm_builds_js(source, base, ci_registry_image, cache_tag)) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index cda17c1e..81c950dd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,6 +193,11 @@ test_dart_format: variables: earthly_target: test-dart-format +test_readme_sync: + extends: .unit_test + variables: + earthly_target: test-readme-sync + # test_check_wasm_builds_dart: # extends: .unit_test # variables: @@ -353,6 +358,8 @@ publish_crates: script: - cargo publish -p veilid-tools --dry-run - cargo publish -p veilid-tools + - cargo publish -p veilid-core-optimized --dry-run + - cargo publish -p veilid-core-optimized - cargo publish -p veilid-core --dry-run - cargo publish -p veilid-core - cargo publish -p veilid-remote-api --dry-run diff --git a/CHANGELOG.md b/CHANGELOG.md index ab5d34da..d748778d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ **UNRELEASED** +- veilid-core-optimized: + - new crate added to hold monomorphic exports for crates that want to be fully speed optimized in all build profiles + - veilid-core: - Stats collection, aggregation, and analysis for all subsystems. Robot-generated python script for reviewing the effect of optimizations. - Fix OnlineDetector spurious flap. diff --git a/Cargo.lock b/Cargo.lock index 40b1d95c..7d630255 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -23,7 +23,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ - "bytes", "crypto-common 0.2.2", "inout 0.2.2", ] @@ -3205,9 +3204,9 @@ checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154" [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" [[package]] name = "macro_rules_attribute" @@ -5893,27 +5892,19 @@ name = "veilid-core" version = "0.5.7" dependencies = [ "arc-swap", - "argon2", "async-std", "async-trait", "async-tungstenite 0.34.1", "backtrace", - "blake2", - "blake3", "bosion", "bytesize", "capnp", "capnpc", "cfg-if", - "chacha20", - "chacha20poly1305", "clap", "console_error_panic_hook", - "curve25519-dalek", "data-encoding", - "digest 0.11.3", "directories", - "ed25519-dalek", "enumset", "eyre", "flume", @@ -5923,7 +5914,6 @@ dependencies = [ "glob", "gloo-events", "hex", - "hpke", "indent", "jni", "js-sys", @@ -5933,7 +5923,6 @@ dependencies = [ "keyvaluedb-web", "lazy_static", "libc", - "lz4_flex", "macro_rules_attribute", "maxminddb", "num-traits", @@ -5949,7 +5938,6 @@ dependencies = [ "serde_ignored", "serde_json", "sha2 0.11.0", - "subtle", "thiserror 2.0.20", "tokio", "tracing", @@ -5957,6 +5945,7 @@ dependencies = [ "tracing-subscriber", "tsify", "tsify-macros", + "veilid-core-optimized", "veilid-hashlink", "veilid-tools", "wasm-bindgen", @@ -5966,7 +5955,6 @@ dependencies = [ "web-sys", "webpki-roots", "ws_stream_wasm", - "x25519-dalek", "zeroize", ] @@ -5991,6 +5979,25 @@ dependencies = [ "veilid-core", ] +[[package]] +name = "veilid-core-optimized" +version = "0.5.7" +dependencies = [ + "argon2", + "blake2", + "blake3", + "chacha20", + "chacha20poly1305", + "curve25519-dalek", + "ed25519-dalek", + "getrandom 0.4.3", + "hpke", + "lz4_flex", + "rand_core 0.10.1", + "subtle", + "x25519-dalek", +] + [[package]] name = "veilid-flutter" version = "0.5.7" diff --git a/Cargo.toml b/Cargo.toml index 381d5507..d7f3e5b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] members = [ "veilid-tools", + "veilid-core-optimized", "veilid-core", "veilid-core/examples/basic", "veilid-core/examples/private_route", @@ -27,13 +28,17 @@ rust-version = "1.89.0" # keyvaluedb-sqlite = { path = "../keyvaluedb/keyvaluedb-sqlite" } # keyvaluedb-web = { path = "../keyvaluedb/keyvaluedb-web" } -# Size-optimized overall; crypto packages below stay at full opt for throughput +# Size-optimized overall; crypto packages below stay at full opt for throughput. A package +# override reaches only that package's own code: generic and #[inline] code is compiled +# where it is instantiated, which is why veilid-core-optimized exists and is listed too. [profile.release] opt-level = "z" lto = true panic = "abort" codegen-units = 1 +[profile.release.package.veilid-core-optimized] +opt-level = 3 [profile.release.package.curve25519-dalek] opt-level = 3 [profile.release.package.ed25519-dalek] @@ -72,6 +77,11 @@ opt-level = 3 [profile.dev.package.backtrace] opt-level = 3 +[profile.dev.package.veilid-core-optimized] +opt-level = 3 +overflow-checks = false +debug-assertions = false + [profile.dev.package.argon2] opt-level = 3 overflow-checks = false diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 7055860a..63058db4 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -38,6 +38,7 @@ Veilid can be built for a number of platforms and targets. Here is a non-exhaust |---------------------|------|-------------------------------------------------------------------| | `veilid-core` | Rust | ✅ [Crates.io](https://crates.io/crates/veilid-core) | | `veilid-tools` | Rust | ✅ [Crates.io](https://crates.io/crates/veilid-tools) | +| `veilid-core-optimized` | Rust | ✅ [Crates.io](https://crates.io/crates/veilid-core-optimized) | | `veilid-remote-api` | Rust | ✅ [Crates.io](https://crates.io/crates/veilid-remote-api) | | `veilid-wasm` | Rust | ❌ Not published directly | | | Typescript | 🟡 [NPM](https://www.npmjs.com/package/@bmv437/veilid-wasm) | diff --git a/Earthfile b/Earthfile index 2c4834b8..87ec3302 100644 --- a/Earthfile +++ b/Earthfile @@ -138,6 +138,9 @@ deps-rust: # Public API checking tool RUN cargo binstall cargo-public-api --disable-strategies=compile --targets $DEFAULT_CARGO_MUSL_TARGET || \ cargo install cargo-public-api --locked + # README sync checking tool + RUN cargo binstall cargo-sync-readme --disable-strategies=compile --targets $DEFAULT_CARGO_MUSL_TARGET || \ + cargo install cargo-sync-readme --locked # Census gate engine. Built from source deliberately: ast-grep ships no musl artifact and # its linux-gnu release targets a newer glibc than this base image, so binstall would # fetch a binary that cannot run here. @@ -222,13 +225,13 @@ deps-flutter: # Code + Flutter deps code-flutter: FROM +deps-flutter - COPY --keep-ts --dir .cargo .dagger dev-setup scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm veilid-remote-api Cargo.lock Cargo.toml clippy.toml sgconfig.yml /veilid + COPY --keep-ts --dir .cargo .dagger dev-setup scripts veilid-cli veilid-core veilid-server veilid-tools veilid-core-optimized veilid-flutter veilid-wasm veilid-remote-api Cargo.lock Cargo.toml clippy.toml sgconfig.yml /veilid WORKDIR /veilid # Make a cache image with downloaded and built dependencies build-linux-cache: FROM +deps-linux - RUN mkdir -p veilid-cli veilid-core veilid-core/examples/basic veilid-core/examples/private_route veilid-server veilid-tools veilid-wasm veilid-flutter veilid-flutter/rust veilid-remote-api + RUN mkdir -p veilid-cli veilid-core veilid-core/examples/basic veilid-core/examples/private_route veilid-server veilid-tools veilid-core-optimized veilid-wasm veilid-flutter veilid-flutter/rust veilid-remote-api COPY --keep-ts --dir .cargo scripts Cargo.lock Cargo.toml . COPY --keep-ts veilid-cli/Cargo.toml veilid-cli COPY --keep-ts veilid-core/Cargo.toml veilid-core @@ -236,15 +239,16 @@ build-linux-cache: COPY --keep-ts veilid-core/examples/private_route/Cargo.toml veilid-core/examples/private_route COPY --keep-ts veilid-server/Cargo.toml veilid-server COPY --keep-ts veilid-tools/Cargo.toml veilid-tools + COPY --keep-ts veilid-core-optimized/Cargo.toml veilid-core-optimized COPY --keep-ts veilid-remote-api/Cargo.toml veilid-remote-api COPY --keep-ts veilid-flutter/rust/Cargo.toml veilid-flutter/rust COPY --keep-ts veilid-wasm/Cargo.toml veilid-wasm RUN cargo chef prepare --recipe-path recipe.json - RUN cargo chef cook --recipe-path recipe.json --profile=test --tests --target $DEFAULT_CARGO_TARGET --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api - RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target x86_64-unknown-linux-gnu --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api - RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target aarch64-unknown-linux-gnu --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api - RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target x86_64-pc-windows-gnu --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api - RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target aarch64-apple-darwin --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api + RUN cargo chef cook --recipe-path recipe.json --profile=test --tests --target $DEFAULT_CARGO_TARGET --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api + RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target x86_64-unknown-linux-gnu --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api + RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target aarch64-unknown-linux-gnu --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api + RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target x86_64-pc-windows-gnu --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api + RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target aarch64-apple-darwin --all-targets --locked -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target wasm32-unknown-unknown --all-targets --locked -p veilid-wasm RUN cargo chef cook --recipe-path recipe.json --zigbuild=clippy --target wasm32-unknown-unknown --all-targets --locked --no-default-features --features=default-wasm -p veilid-flutter ARG CI_REGISTRY_IMAGE=registry.gitlab.com/veilid/veilid @@ -270,7 +274,7 @@ code-linux: RUN echo "Using cache from registry: $CI_REGISTRY_IMAGE/$CACHE_NAME:$CACHE_TAG" FROM $CI_REGISTRY_IMAGE/$CACHE_NAME:$CACHE_TAG END - COPY --keep-ts --dir .cargo scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm veilid-remote-api Cargo.lock Cargo.toml clippy.toml sgconfig.yml /veilid + COPY --keep-ts --dir .cargo scripts veilid-cli veilid-core veilid-server veilid-tools veilid-core-optimized veilid-flutter veilid-wasm veilid-remote-api Cargo.lock Cargo.toml clippy.toml sgconfig.yml /veilid # Check to make sure Cargo.lock is up to date RUN cargo update -w --locked # Restore original Cargo.lock @@ -283,7 +287,7 @@ code-linux: # Code + Linux + Android deps code-android: FROM +deps-android - COPY --keep-ts --dir .cargo scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm veilid-remote-api Cargo.lock Cargo.toml clippy.toml sgconfig.yml /veilid + COPY --keep-ts --dir .cargo scripts veilid-cli veilid-core veilid-server veilid-tools veilid-core-optimized veilid-flutter veilid-wasm veilid-remote-api Cargo.lock Cargo.toml clippy.toml sgconfig.yml /veilid COPY --keep-ts scripts/earthly/cargo-android/config.toml /veilid/.cargo/config.toml # Lints only @@ -320,7 +324,7 @@ build-linux-amd64: RUN echo "not enough container memory to build. increase build host memory." RUN false END - RUN cargo zigbuild --locked --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api + RUN cargo zigbuild --locked --target x86_64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api SAVE ARTIFACT ./target/x86_64-unknown-linux-gnu AS LOCAL ./target/artifacts/x86_64-unknown-linux-gnu build-linux-arm64: @@ -330,7 +334,7 @@ build-linux-arm64: RUN echo "not enough container memory to build. increase build host memory." RUN false END - RUN cargo zigbuild --locked --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api + RUN cargo zigbuild --locked --target aarch64-unknown-linux-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api SAVE ARTIFACT ./target/aarch64-unknown-linux-gnu AS LOCAL ./target/artifacts/aarch64-unknown-linux-gnu # build-windows-amd64: @@ -340,7 +344,7 @@ build-linux-arm64: # RUN echo "not enough container memory to build. increase build host memory." # RUN false # END -# RUN cargo zigbuild --locked --target x86_64-pc-windows-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api +# RUN cargo zigbuild --locked --target x86_64-pc-windows-gnu --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api # SAVE ARTIFACT ./target/x86_64-pc-windows-gnu AS LOCAL ./target/artifacts/x86_64-pc-windows-gnu # build-macos-arm64: @@ -350,7 +354,7 @@ build-linux-arm64: # RUN echo "not enough container memory to build. increase build host memory." # RUN false # END -# RUN cargo zigbuild --locked --target aarch64-apple-darwin --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api +# RUN cargo zigbuild --locked --target aarch64-apple-darwin --release -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api # SAVE ARTIFACT ./target/aarch64-apple-darwin AS LOCAL ./target/artifacts/aarch64-apple-darwin @@ -421,12 +425,16 @@ test-unit-tests-all: test-dart-format: FROM +code-flutter RUN ./scripts/check_format.sh +test-readme-sync: + FROM +code-linux + RUN ./scripts/check_readme_sync.sh test-flutter-integration: FROM +code-flutter RUN ./scripts/_flutter_integration_tests.sh test-all: BUILD +test-census BUILD +test-dart-format + BUILD +test-readme-sync BUILD +test-msrv-all BUILD +test-smoketest-all BUILD +test-public-api diff --git a/RELEASING.md b/RELEASING.md index 6a851864..79d14a72 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -45,10 +45,12 @@ Git tags serve as a historical record of what repo versions were released at whi 1. Configure the crates.io credentials, if not already accomplished. 2. Execute `cargo publish -p veilid-tools --dry-run` 3. Execute `cargo publish -p veilid-tools` -4. Execute `cargo publish -p veilid-core --dry-run` -5. Execute `cargo publish -p veilid-core` -6. Execute `cargo publish -p veilid-remote-api --dry-run` -7. Execute `cargo publish -p veilid-remote-api` +4. Execute `cargo publish -p veilid-core-optimized --dry-run` +5. Execute `cargo publish -p veilid-core-optimized` +6. Execute `cargo publish -p veilid-core --dry-run` +7. Execute `cargo publish -p veilid-core` +8. Execute `cargo publish -p veilid-remote-api --dry-run` +9. Execute `cargo publish -p veilid-remote-api` ## Publish to Pypi (Manual steps if needed) @@ -68,6 +70,9 @@ Occasionally a release will happen that needs to be reverted. This is done manua - [x] __veilid-tools__ [__Tag__: `veilid-tools-v0.0.0`] > An assortment of useful components used by the other Veilid crates. > Released to crates.io when its version number is changed in `Cargo.toml` +- [x] __veilid-core-optimized__ [__Tag__: `veilid-core-optimized-v0.0.0`] + > Monomorphic entry points for veilid-core's crypto and compression dependencies, built at full optimization in every profile + > Released to crates.io when its version number is changed in `Cargo.toml` - [x] __veilid-core__ [__Tag__: `veilid-core-v0.0.0`] > The base rust crate for Veilid's logic > Released to crates.io when its version number is changed in `Cargo.toml` diff --git a/dev-setup/setup_linux.sh b/dev-setup/setup_linux.sh index 346a88a0..6e3c4ab9 100755 --- a/dev-setup/setup_linux.sh +++ b/dev-setup/setup_linux.sh @@ -123,6 +123,7 @@ i_wbg() { cargo install --locked wasm-bindgen-cli --version "$REQ_WASMBINDGEN" i_cedit() { cargo install --locked cargo-edit --version "$REQ_CARGOEDIT"; } i_nxt() { cargo install --locked cargo-nextest; } i_papi() { cargo install --locked cargo-public-api; } +i_srm() { cargo binstall --no-confirm cargo-sync-readme 2>/dev/null || cargo install --locked cargo-sync-readme; } i_sg() { cargo install --locked ast-grep --version "$REQ_ASTGREP"; } crate_ver() { cargo install --list 2>/dev/null | sed -n "s/^$1 v\([0-9.]*\):.*/\1/p" | head -1; } @@ -145,6 +146,7 @@ if [ "$BUILDONLY" != 1 ]; then dep_exact "cargo-edit" "$(crate_ver cargo-edit)" "$REQ_CARGOEDIT" 1 i_cedit dep_have "cargo-nextest" "$(crate_ver cargo-nextest)" "cargo install --locked cargo-nextest" i_nxt dep_have "cargo-public-api" "$(crate_ver cargo-public-api)" "cargo install --locked cargo-public-api" i_papi + dep_have "cargo-sync-readme" "$(crate_ver cargo-sync-readme)" "cargo binstall cargo-sync-readme || cargo install --locked cargo-sync-readme" i_srm dep_exact "ast-grep" "$(crate_ver ast-grep)" "$REQ_ASTGREP" 1 i_sg fi diff --git a/dev-setup/setup_macos.sh b/dev-setup/setup_macos.sh index b436a282..15d6c90b 100755 --- a/dev-setup/setup_macos.sh +++ b/dev-setup/setup_macos.sh @@ -102,6 +102,7 @@ i_wbg() { cargo install --locked wasm-bindgen-cli --version "$REQ_WASMBINDGEN" i_cedit() { cargo install --locked cargo-edit --version "$REQ_CARGOEDIT"; } i_nxt() { cargo install --locked cargo-nextest; } i_papi() { cargo install --locked cargo-public-api; } +i_srm() { cargo binstall --no-confirm cargo-sync-readme 2>/dev/null || cargo install --locked cargo-sync-readme; } i_sg() { cargo install --locked ast-grep --version "$REQ_ASTGREP"; } crate_ver() { cargo install --list 2>/dev/null | sed -n "s/^$1 v\([0-9.]*\):.*/\1/p" | head -1; } @@ -129,6 +130,7 @@ if [ "$BUILDONLY" != 1 ]; then dep_exact "cargo-edit" "$(crate_ver cargo-edit)" "$REQ_CARGOEDIT" 1 i_cedit dep_have "cargo-nextest" "$(crate_ver cargo-nextest)" "cargo install --locked cargo-nextest" i_nxt dep_have "cargo-public-api" "$(crate_ver cargo-public-api)" "cargo install --locked cargo-public-api" i_papi + dep_have "cargo-sync-readme" "$(crate_ver cargo-sync-readme)" "cargo binstall cargo-sync-readme || cargo install --locked cargo-sync-readme" i_srm dep_exact "ast-grep" "$(crate_ver ast-grep)" "$REQ_ASTGREP" 1 i_sg fi diff --git a/dev-setup/setup_windows.ps1 b/dev-setup/setup_windows.ps1 index 33048aac..9319a99e 100644 --- a/dev-setup/setup_windows.ps1 +++ b/dev-setup/setup_windows.ps1 @@ -293,6 +293,7 @@ if (-not $BuildOnly) { DepExact "cargo-edit" (CrateVer "cargo-edit") $V.CARGO_EDIT_VERSION $true $iCedit DepHave "cargo-nextest" (CrateVer "cargo-nextest") "cargo install --locked cargo-nextest" { cargo install --locked cargo-nextest } DepHave "cargo-public-api" (CrateVer "cargo-public-api") "cargo install --locked cargo-public-api" { cargo install --locked cargo-public-api } + DepHave "cargo-sync-readme" (CrateVer "cargo-sync-readme") "cargo binstall cargo-sync-readme || cargo install --locked cargo-sync-readme" { cargo binstall --no-confirm cargo-sync-readme 2>$null; if ($LASTEXITCODE -ne 0) { cargo install --locked cargo-sync-readme } } } # JDK for the jni/jvm link. Provisioned (download + JAVA_HOME, keyed off -TargetArch) whenever we diff --git a/scripts/_build_docs.bat b/scripts/_build_docs.bat index a698efe5..39c662d1 100644 --- a/scripts/_build_docs.bat +++ b/scripts/_build_docs.bat @@ -20,6 +20,7 @@ if "%TOOLCHAIN%"=="" ( cargo test --doc cargo +%TOOLCHAIN% docs-rs -p veilid-core cargo +%TOOLCHAIN% docs-rs -p veilid-tools +cargo +%TOOLCHAIN% docs-rs -p veilid-core-optimized cargo +%TOOLCHAIN% docs-rs -p veilid-remote-api POPD diff --git a/scripts/_build_docs.sh b/scripts/_build_docs.sh index c76c12a1..5294c346 100755 --- a/scripts/_build_docs.sh +++ b/scripts/_build_docs.sh @@ -36,6 +36,11 @@ if [ $CLEANUP -eq 1 ] ; then rm -rf target/$TARGET/doc fi +cargo +$TOOLCHAIN docs-rs -p veilid-core-optimized --target $TARGET --locked +if [ $CLEANUP -eq 1 ] ; then + rm -rf target/$TARGET/doc +fi + cargo +$TOOLCHAIN docs-rs -p veilid-remote-api --target $TARGET --locked if [ $CLEANUP -eq 1 ] ; then rm -rf target/$TARGET/doc diff --git a/scripts/_msrv_all.sh b/scripts/_msrv_all.sh index 96b7c34b..49f6d3b3 100755 --- a/scripts/_msrv_all.sh +++ b/scripts/_msrv_all.sh @@ -12,6 +12,9 @@ fi #echo "Checking MSRV for veilid-tools" #cargo msrv verify --manifest-path=veilid-tools/Cargo.toml +#echo "Checking MSRV for veilid-core-optimized" +#cargo msrv verify --manifest-path=veilid-core-optimized/Cargo.toml + #echo "Checking MSRV for veilid-core" #cargo msrv verify --manifest-path=veilid-core/Cargo.toml diff --git a/scripts/_readme_crates.sh b/scripts/_readme_crates.sh new file mode 100644 index 00000000..70a0fca6 --- /dev/null +++ b/scripts/_readme_crates.sh @@ -0,0 +1,7 @@ +# The crate directories whose README.md is generated from the crate docs by cargo-sync-readme, +# found by their markers so a new crate joins the gate by carrying them. +# Sourced by scripts/check_readme_sync.sh and scripts/fmt.sh from the repository root. + +readme_crates() { + grep -l "cargo-sync-readme start" -- */README.md 2>/dev/null | xargs -n1 dirname | sort +} diff --git a/scripts/_unit_tests_all.sh b/scripts/_unit_tests_all.sh index 3329b535..03792f0d 100755 --- a/scripts/_unit_tests_all.sh +++ b/scripts/_unit_tests_all.sh @@ -28,7 +28,7 @@ else # network. The mock pass below covers the attachment lifecycle and run_test_resolver covers # the resolver against a loopback nameserver, so both are excluded to keep this script offline. echo "Skipping run_test_attachment_manager and run_test_resolver_live (need the network)" - cargo nextest run --tests --target $TARGET --all-targets --locked --features=debug-locks -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core -p veilid-remote-api \ + cargo nextest run --tests --target $TARGET --all-targets --locked --features=debug-locks -p veilid-server -p veilid-cli -p veilid-tools -p veilid-core-optimized -p veilid-core -p veilid-remote-api \ -E 'not (test(=tests::native::run_test_attachment_manager) + test(=tests::native::run_test_resolver_live))' status=$? diff --git a/scripts/census/rules/panic-census-opaque.yml b/scripts/census/rules/panic-census-opaque.yml index 8e5217c8..3bb5eaf3 100644 --- a/scripts/census/rules/panic-census-opaque.yml +++ b/scripts/census/rules/panic-census-opaque.yml @@ -12,6 +12,7 @@ note: | lands in the census, and a new panic added inside a cfg_if! shows up as a census diff rather than as nothing at all. files: + - "veilid-core-optimized/src/**" - "veilid-core/src/**" - "veilid-tools/src/**" ignores: diff --git a/scripts/census/rules/panic-census-sites.yml b/scripts/census/rules/panic-census-sites.yml index 679922d5..e21af108 100644 --- a/scripts/census/rules/panic-census-sites.yml +++ b/scripts/census/rules/panic-census-sites.yml @@ -7,6 +7,7 @@ note: | the empty-sweep backstop: if the matcher silently stops working this count collapses to zero and the census diff catches it. files: + - "veilid-core-optimized/src/**" - "veilid-core/src/**" - "veilid-tools/src/**" ignores: diff --git a/scripts/census/rules/panic-census.yml b/scripts/census/rules/panic-census.yml index 9e081ee1..75ea654f 100644 --- a/scripts/census/rules/panic-census.yml +++ b/scripts/census/rules/panic-census.yml @@ -20,6 +20,7 @@ note: | not something to declare. todo! is out for a different reason - it marks code not written yet, which the compiler already tracks. files: + - "veilid-core-optimized/src/**" - "veilid-core/src/**" - "veilid-tools/src/**" ignores: diff --git a/scripts/census/rules/unsafe-safety-sites.yml b/scripts/census/rules/unsafe-safety-sites.yml index e957852f..657de685 100644 --- a/scripts/census/rules/unsafe-safety-sites.yml +++ b/scripts/census/rules/unsafe-safety-sites.yml @@ -8,6 +8,7 @@ note: | this count collapses to zero and the census diff catches it. A rule that matches nothing passes forever otherwise. files: + - "veilid-core-optimized/src/**" - "veilid-core/src/**" - "veilid-tools/src/**" utils: diff --git a/scripts/census/rules/unsafe-safety.yml b/scripts/census/rules/unsafe-safety.yml index db77612b..b993b561 100644 --- a/scripts/census/rules/unsafe-safety.yml +++ b/scripts/census/rules/unsafe-safety.yml @@ -12,6 +12,7 @@ note: | A comment serves exactly the one block below it, so two adjacent unsafe blocks need two comments. files: + - "veilid-core-optimized/src/**" - "veilid-core/src/**" - "veilid-tools/src/**" diff --git a/scripts/check_all.sh b/scripts/check_all.sh index 0ff139cc..0ce6b1a0 100755 --- a/scripts/check_all.sh +++ b/scripts/check_all.sh @@ -99,6 +99,10 @@ fi echo "Performing Dart format check..." step ./scripts/check_format.sh +# Crate READMEs are generated from the crate docs; same script CI's test_readme_sync job runs. +echo "Performing README sync check..." +step ./scripts/check_readme_sync.sh + # Dagger's ignore patterns are generated from the repo's .gitignore files. Regenerating belongs # with a push, not with CI, which has nothing to commit the result to. if ! command -v uv > /dev/null 2>&1; then diff --git a/scripts/check_readme_sync.sh b/scripts/check_readme_sync.sh new file mode 100755 index 00000000..59e6766a --- /dev/null +++ b/scripts/check_readme_sync.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# README gate: every crate README carrying cargo-sync-readme markers must match its crate +# documentation. scripts/fmt.sh regenerates them. +set -euo pipefail + +cd "$(dirname "$0")/.." +source scripts/_readme_crates.sh + +if ! cargo sync-readme --version >/dev/null 2>&1; then + echo "readme gate FAILED: cargo-sync-readme is not installed. Install it with:" >&2 + echo " cargo binstall cargo-sync-readme || cargo install --locked cargo-sync-readme" >&2 + exit 1 +fi + +crates=$(readme_crates) +# A sweep that sees no crates is a broken sweep, not a clean one +if [ -z "$crates" ]; then + echo "readme gate FAILED: no crate README carries cargo-sync-readme markers." >&2 + exit 1 +fi + +failed=0 +for crate in $crates; do + if ! (cd "$crate" && cargo sync-readme --check >/dev/null 2>&1); then + echo "readme gate: $crate/README.md is out of sync with its crate documentation" >&2 + failed=1 + fi +done +if [ "$failed" -ne 0 ]; then + echo >&2 + echo "readme gate FAILED: run ./scripts/fmt.sh and commit the result." >&2 + exit 1 +fi diff --git a/scripts/cicd/cicd-infra-description.md b/scripts/cicd/cicd-infra-description.md index 30c6f8b5..930a72cd 100644 --- a/scripts/cicd/cicd-infra-description.md +++ b/scripts/cicd/cicd-infra-description.md @@ -11,7 +11,7 @@ The droplets are divided into three categories: build machines, build orchestrat * Enrolled with Gitlab Runner * Earthly installed * Droplet size: c2-8vcpu-16gb-intel - * The amd64-deb build machine also builds and uploads the veilid-core and veilid-tools Rust crates and veilid-python module to crates.io and Pypi + * The amd64-deb build machine also builds and uploads the veilid-tools, veilid-core-optimized, veilid-core and veilid-remote-api Rust crates and the veilid-python module to crates.io and Pypi * SCPs compiled packages to the orchestration machine ovber private networking * Build Orchestration * Single Debian machine with minimal resources @@ -29,7 +29,7 @@ The droplets are divided into three categories: build machines, build orchestrat 2. Gitlab CICD builds a SaaS container in which Earthly tests are performed on the latest version of the Main branch. A test fail will exit the CICD process. 3. The Gitlab Runner registered to the build orchestration machine executes the build-machine-ctl.sh script to create the build machines. 4. The Gitlab Runners registered to the build machines execute their specified arch/OS build as defined in the Earthly execution command in .gitlab-ci.yml. -5. The build machine for amd64-deb additionally compiles veilid-core and veilid-tools Rust crates and the veilid-python module. These are uploaded to crates.io and Pypi as part of their respective build processes. +5. The build machine for amd64-deb additionally compiles the veilid-tools, veilid-core-optimized, veilid-core and veilid-remote-api Rust crates (published in that order, each depending on the last) and the veilid-python module. These are uploaded to crates.io and Pypi as part of their respective build processes. 6. When a build completes, the Gitlab Runner then executes the scp-to-orchestrator.sh script when sends the .deb or .rpm packages to build orchestration. 7. Once the build jobs in CICD have completed, the Gitlab Runner registered to build orchestration executes the distribute-packages.sh script which results in signed .deb and .rpm repositories being sent to the repo server. 8. The build orchecstration machine sends droplet delete commands to Digital Ocean for each of the build machines. diff --git a/scripts/deprecated/debug_main_node.sh b/scripts/deprecated/debug_main_node.sh deleted file mode 100755 index 22a99e2a..00000000 --- a/scripts/deprecated/debug_main_node.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -exec ./run_local_test.py 20 -w 0 --config-file ./local-test.yml $1 diff --git a/scripts/deprecated/debug_subnode_1.sh b/scripts/deprecated/debug_subnode_1.sh deleted file mode 100755 index 6274ed59..00000000 --- a/scripts/deprecated/debug_subnode_1.sh +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/bash -exec ./run_local_test.py 2 -w 1 --log_trace --config-file ./local-test.yml diff --git a/scripts/deprecated/local-test.yml b/scripts/deprecated/local-test.yml deleted file mode 100644 index 6e73ef0c..00000000 --- a/scripts/deprecated/local-test.yml +++ /dev/null @@ -1,6 +0,0 @@ ---- -core: - network: - dht: - min_peer_count: 1 - bootstrap: [] diff --git a/scripts/deprecated/run_2.sh b/scripts/deprecated/run_2.sh deleted file mode 100755 index 1f786376..00000000 --- a/scripts/deprecated/run_2.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -exec ./run_local_test.py 2 --config-file ./local-test.yml $@ - - diff --git a/scripts/deprecated/run_20.sh b/scripts/deprecated/run_20.sh deleted file mode 100755 index 21022a7d..00000000 --- a/scripts/deprecated/run_20.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -exec ./run_local_test.py 20 --config-file ./local-test.yml $@ - - diff --git a/scripts/deprecated/run_3.sh b/scripts/deprecated/run_3.sh deleted file mode 100755 index 04d1a01e..00000000 --- a/scripts/deprecated/run_3.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -exec ./run_local_test.py 3 --config-file ./local-test.yml $@ - - diff --git a/scripts/deprecated/run_4.sh b/scripts/deprecated/run_4.sh deleted file mode 100755 index d29e7a0f..00000000 --- a/scripts/deprecated/run_4.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -exec ./run_local_test.py 4 --config-file ./local-test.yml $@ - - diff --git a/scripts/deprecated/run_8.sh b/scripts/deprecated/run_8.sh deleted file mode 100755 index 863f97f2..00000000 --- a/scripts/deprecated/run_8.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -exec ./run_local_test.py 8 --config-file ./local-test.yml $@ - - diff --git a/scripts/deprecated/run_local_test.py b/scripts/deprecated/run_local_test.py deleted file mode 100755 index 3a3216ab..00000000 --- a/scripts/deprecated/run_local_test.py +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env python3 - -import sys -import os -import io -import argparse -import subprocess -import signal -import time -from threading import Thread - -if sys.version_info < (3, 0, 0): - print(__file__ + ' requires Python 3, while Python ' + - str(sys.version[0] + ' was detected. Terminating. ')) - sys.exit(1) - -script_dir = os.path.dirname(os.path.realpath(__file__)) -veilid_server_exe_debug = os.path.join(script_dir, '..', - 'target', 'debug', 'veilid-server') -veilid_server_exe_release = os.path.join( - script_dir, '..', 'target', 'release', 'veilid-server') -main_process = None -subindex_processes = [] - -try: - # Python 3, open as binary, then wrap in a TextIOWrapper with write-through. - sys.stdout = io.TextIOWrapper( - open(sys.stdout.fileno(), 'wb', 0), write_through=True) - sys.stderr = io.TextIOWrapper( - open(sys.stderr.fileno(), 'wb', 0), write_through=True) -except TypeError: - # Python 2 - sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) - sys.stderr = os.fdopen(sys.stderr.fileno(), 'w', 0) - - -def tee(prefix, infile, *files): - """Print `infile` to `files` in a separate thread.""" - - def fanout(prefix, infile, *files): - with infile: - for line in iter(infile.readline, b""): - for f in files: - f.write(prefix + line) - f.flush() - - t = Thread(target=fanout, args=(prefix, infile,) + files) - t.daemon = True - t.start() - return t - - -def read_until_interface_dial_info(proc, proto): - - interface_dial_info_str = b"Local Dial Info: " - for ln in iter(proc.stdout.readline, ""): - sys.stdout.buffer.write(ln) - sys.stdout.flush() - - idx = ln.find(interface_dial_info_str) - if idx != -1: - idx += len(interface_dial_info_str) - di = ln[idx:] - if b"@"+bytes(proto)+b"|" in di: - return di.decode("utf-8").strip() - - return None - - -class CleanChildProcesses: - def __enter__(self): - os.setpgrp() # create new process group, become its leader - - def __exit__(self, type, value, traceback): - try: - os.killpg(0, signal.SIGKILL) # kill all processes in my group - except KeyboardInterrupt: - pass - - -def main(): - threads = [] - - # Parse arguments - parser = argparse.ArgumentParser(description='Run veilid servers locally') - parser.add_argument("count", type=int, - help='number of instances to run') - parser.add_argument("--release", action='store_true', - help='use release mode build') - parser.add_argument("--log_trace", action='store_true', - help='use trace logging') - parser.add_argument("--log_info", action='store_true', - help='use info logging') - parser.add_argument("-w", "--wait-for-debug", action='append', - help='specify subnode index to wait for the debugger') - parser.add_argument("--config-file", type=str, - help='configuration file to specify for the bootstrap node') - parser.add_argument("--protocol", type=str, default="udp", - help='default protocol to choose for dial info') - args = parser.parse_args() - - if args.count < 1: - print("Must specify more than one instance") - sys.exit(1) - - veilid_server_exe = None - if args.release: - veilid_server_exe = veilid_server_exe_release - else: - veilid_server_exe = veilid_server_exe_debug - - base_args = [veilid_server_exe] - if args.log_info: - pass - elif args.log_trace: - base_args.append("--trace") - else: - base_args.append("--debug") - - if args.config_file: - base_args.append("--config-file={}".format(args.config_file)) - - # Run primary node and get node id - main_args = base_args.copy() - if args.wait_for_debug and ("0" in args.wait_for_debug): - main_args.append("--wait-for-debug") - - print("Running main node: {}".format(str(main_args))) - main_proc = subprocess.Popen( - main_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - print(">>> MAIN NODE PID={}".format(main_proc.pid)) - - main_di = read_until_interface_dial_info( - main_proc, bytes(args.protocol, 'utf-8')) - - print(">>> MAIN DIAL INFO={}".format(main_di)) - - threads.append( - tee(b"Veilid-0: ", main_proc.stdout, open("/tmp/veilid-0-out", "wb"), - getattr(sys.stdout, "buffer", sys.stdout)) - ) - - # Run all secondaries and add primary to bootstrap - for n in range(1, args.count): - - time.sleep(1) - - sub_args = base_args.copy() - sub_args.append("--subnode-index={}".format(n)) - sub_args.append("--bootstrap-nodes={}".format(main_di)) - if args.wait_for_debug and (str(n) in args.wait_for_debug): - sub_args.append("--wait-for-debug") - - print("Running subnode {}: {}".format(n, str(sub_args))) - - sub_proc = subprocess.Popen( - sub_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - - print(">>> SUBNODE {} NODE PID={}".format(n, sub_proc.pid)) - - threads.append( - tee("Veilid-{}: ".format(n).encode("utf-8"), sub_proc.stdout, open("/tmp/veilid-{}-out".format(n), "wb"), - getattr(sys.stdout, "buffer", sys.stdout)) - ) - - for t in threads: - t.join() # wait for IO completion - - sys.exit(0) - - -if __name__ == "__main__": - - with CleanChildProcesses(): - main() diff --git a/scripts/fmt.sh b/scripts/fmt.sh index cff0884d..519ce0a6 100755 --- a/scripts/fmt.sh +++ b/scripts/fmt.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash -# Format every Dart source scripts/check_format.sh checks, then `cargo fmt` the Rust side. -# Run it, then commit the result. +# Format every Dart source scripts/check_format.sh checks, `cargo fmt` the Rust side, and +# regenerate the crate READMEs scripts/check_readme_sync.sh checks. Run it, then commit the +# result. set -euo pipefail cd "$(dirname "$0")/.." source scripts/_dart_sources.sh +source scripts/_readme_crates.sh files=$(dart_source_files) if [ -z "$files" ]; then @@ -14,3 +16,10 @@ fi echo "$files" | xargs dart format --language-version="$DART_FORMAT_LANGUAGE_VERSION" -- cargo fmt --all + +for crate in $(readme_crates); do + if ! (cd "$crate" && cargo sync-readme --check >/dev/null 2>&1); then + echo "fmt: regenerating $crate/README.md" + (cd "$crate" && cargo sync-readme) + fi +done diff --git a/scripts/msrv_all.sh b/scripts/msrv_all.sh index fa287a51..d294527a 100755 --- a/scripts/msrv_all.sh +++ b/scripts/msrv_all.sh @@ -3,6 +3,7 @@ SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" pushd $SCRIPTDIR/.. >/dev/null cargo msrv verify --manifest-path=veilid-tools/Cargo.toml +cargo msrv verify --manifest-path=veilid-core-optimized/Cargo.toml cargo msrv verify --manifest-path=veilid-core/Cargo.toml cargo msrv verify --manifest-path=veilid-server/Cargo.toml cargo msrv verify --manifest-path=veilid-cli/Cargo.toml diff --git a/scripts/tools/keytool.py b/scripts/tools/keytool.py deleted file mode 100644 index 90323314..00000000 --- a/scripts/tools/keytool.py +++ /dev/null @@ -1,99 +0,0 @@ -import base64 -import sys -import argparse - -def urlsafe_b64encode_no_pad(b: bytes) -> str: - """ - Removes any `=` used as padding from the encoded string. - """ - return base64.urlsafe_b64encode(b).decode().rstrip("=") - - -def urlsafe_b64decode_no_pad(s: str) -> bytes: - """ - Adds back in the required padding before decoding. - """ - padding = 4 - (len(s) % 4) - s = s + ("=" * padding) - return base64.urlsafe_b64decode(s) - - -def do_value(args): - - key = urlsafe_b64decode_no_pad(args.key) - - print("key:", key.hex()) - - -def dist(key1: bytes, key2: bytes) -> bytes: - distance = bytearray(len(key1)) - for n in range(len(key1)): - distance[n] = key1[n] ^ key2[n] - - return bytes(distance) - - -def do_distance(args): - - key1 = urlsafe_b64decode_no_pad(args.key1) - key2 = urlsafe_b64decode_no_pad(args.key2) - - print("key1:", key1.hex()) - print("key2:", key2.hex()) - - distance = dist(key1, key2) - print("dist:", distance.hex()) - -def keycmp(key1: bytes, key2: bytes) -> int: - for n in range(len(key1)): - if key1[n] < key2[n]: - return -1 - if key1[n] > key2[n]: - return 1 - return 0 - -def do_closer(args): - - key = urlsafe_b64decode_no_pad(args.key) - near = urlsafe_b64decode_no_pad(args.near) - far = urlsafe_b64decode_no_pad(args.far) - - print(" key:", key.hex()) - print("near:", near.hex()) - print(" far:", far.hex()) - - distance_near = dist(key, near) - distance_far = dist(key, far) - - print(" dn:", distance_near.hex()) - print(" df:", distance_far.hex()) - - c = keycmp(distance_near, distance_far) - print(" cmp:", c) - -def main(): - parser = argparse.ArgumentParser() - subparsers = parser.add_subparsers(required=True) - - parser_value = subparsers.add_parser('value') - parser_value.add_argument('key', type=str) - parser_value.set_defaults(func=do_value) - - parser_value = subparsers.add_parser('distance') - parser_value.add_argument('key1', type=str) - parser_value.add_argument('key2', type=str) - parser_value.set_defaults(func=do_distance) - - parser_value = subparsers.add_parser('closer') - parser_value.add_argument('key', type=str) - parser_value.add_argument('near', type=str) - parser_value.add_argument('far', type=str) - parser_value.set_defaults(func=do_closer) - - args = parser.parse_args() - args.func(args) - - sys.exit(0) - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/scripts/update_public_api.sh b/scripts/update_public_api.sh index 9649c5e7..684b34f6 100755 --- a/scripts/update_public_api.sh +++ b/scripts/update_public_api.sh @@ -25,25 +25,31 @@ PUBLIC_API_TXT_FILE=public-api.txt PUBLIC_API_TARGET=x86_64-unknown-linux-gnu PUBLIC_API_FILE_VEILID_CORE=veilid-core/$PUBLIC_API_TXT_FILE PUBLIC_API_FILE_VEILID_TOOLS=veilid-tools/$PUBLIC_API_TXT_FILE +PUBLIC_API_FILE_VEILID_CORE_OPTIMIZED=veilid-core-optimized/$PUBLIC_API_TXT_FILE PUBLIC_API_FILE_VEILID_REMOTE_API=veilid-remote-api/$PUBLIC_API_TXT_FILE PUBLIC_API_FEATURES_VEILID_CORE="geolocation,footgun-nodeid-target,footgun-config" +PUBLIC_API_FEATURES_VEILID_CORE_OPTIMIZED="enable-crypto-vld0,enable-crypto-none" # Make temporary file for the public API output PUBLIC_API_TMP_FILE_VEILID_CORE=$(mktemp) || exit 2 PUBLIC_API_TMP_FILE_VEILID_TOOLS=$(mktemp) || exit 2 +PUBLIC_API_TMP_FILE_VEILID_CORE_OPTIMIZED=$(mktemp) || exit 2 PUBLIC_API_TMP_FILE_VEILID_REMOTE_API=$(mktemp) || exit 2 PUBLIC_API_TMP_FILE_VEILID_CORE_DIFF=$(mktemp) || exit 2 PUBLIC_API_TMP_FILE_VEILID_TOOLS_DIFF=$(mktemp) || exit 2 +PUBLIC_API_TMP_FILE_VEILID_CORE_OPTIMIZED_DIFF=$(mktemp) || exit 2 PUBLIC_API_TMP_FILE_VEILID_REMOTE_API_DIFF=$(mktemp) || exit 2 # Ensure cleanup happens on exit, interruption, or termination -trap "rm -f $PUBLIC_API_TMP_FILE_VEILID_CORE $PUBLIC_API_TMP_FILE_VEILID_CORE_DIFF $PUBLIC_API_TMP_FILE_VEILID_TOOLS $PUBLIC_API_TMP_FILE_VEILID_TOOLS_DIFF $PUBLIC_API_TMP_FILE_VEILID_REMOTE_API $PUBLIC_API_TMP_FILE_VEILID_REMOTE_API_DIFF" EXIT +trap "rm -f $PUBLIC_API_TMP_FILE_VEILID_CORE $PUBLIC_API_TMP_FILE_VEILID_CORE_DIFF $PUBLIC_API_TMP_FILE_VEILID_TOOLS $PUBLIC_API_TMP_FILE_VEILID_TOOLS_DIFF $PUBLIC_API_TMP_FILE_VEILID_CORE_OPTIMIZED $PUBLIC_API_TMP_FILE_VEILID_CORE_OPTIMIZED_DIFF $PUBLIC_API_TMP_FILE_VEILID_REMOTE_API $PUBLIC_API_TMP_FILE_VEILID_REMOTE_API_DIFF" EXIT # Run cargo public-api and save the output to the temporary file echo "Checking public API for 'veilid-core'..." cargo public-api -sss -p veilid-core --target $PUBLIC_API_TARGET --color never --features $PUBLIC_API_FEATURES_VEILID_CORE > $PUBLIC_API_TMP_FILE_VEILID_CORE echo "Checking public API for 'veilid-tools'..." cargo public-api -sss -p veilid-tools --target $PUBLIC_API_TARGET --color never > $PUBLIC_API_TMP_FILE_VEILID_TOOLS +echo "Checking public API for 'veilid-core-optimized'..." +cargo public-api -sss -p veilid-core-optimized --target $PUBLIC_API_TARGET --color never --features $PUBLIC_API_FEATURES_VEILID_CORE_OPTIMIZED > $PUBLIC_API_TMP_FILE_VEILID_CORE_OPTIMIZED echo "Checking public API for 'veilid-remote-api'..." cargo public-api -sss -p veilid-remote-api --target $PUBLIC_API_TARGET --color never > $PUBLIC_API_TMP_FILE_VEILID_REMOTE_API @@ -72,10 +78,12 @@ diff_and_print "veilid-core" $PUBLIC_API_FILE_VEILID_CORE $PUBLIC_API_TMP_FILE_V veilid_core_diff_status=$? diff_and_print "veilid-tools" $PUBLIC_API_FILE_VEILID_TOOLS $PUBLIC_API_TMP_FILE_VEILID_TOOLS $PUBLIC_API_TMP_FILE_VEILID_TOOLS_DIFF veilid_tools_diff_status=$? +diff_and_print "veilid-core-optimized" $PUBLIC_API_FILE_VEILID_CORE_OPTIMIZED $PUBLIC_API_TMP_FILE_VEILID_CORE_OPTIMIZED $PUBLIC_API_TMP_FILE_VEILID_CORE_OPTIMIZED_DIFF +veilid_core_optimized_diff_status=$? diff_and_print "veilid-remote-api" $PUBLIC_API_FILE_VEILID_REMOTE_API $PUBLIC_API_TMP_FILE_VEILID_REMOTE_API $PUBLIC_API_TMP_FILE_VEILID_REMOTE_API_DIFF veilid_remote_api_diff_status=$? -if [[ "$veilid_core_diff_status" -ne 0 || "$veilid_tools_diff_status" -ne 0 || "$veilid_remote_api_diff_status" -ne 0 ]]; then +if [[ "$veilid_core_diff_status" -ne 0 || "$veilid_tools_diff_status" -ne 0 || "$veilid_core_optimized_diff_status" -ne 0 || "$veilid_remote_api_diff_status" -ne 0 ]]; then all_diff_status=1 else all_diff_status=0 @@ -104,6 +112,10 @@ if [ $all_diff_status -eq 1 ]; then mv $PUBLIC_API_TMP_FILE_VEILID_TOOLS $PUBLIC_API_FILE_VEILID_TOOLS echo "Public API updated for 'veilid-tools'." fi + if [ $veilid_core_optimized_diff_status -eq 1 ]; then + mv $PUBLIC_API_TMP_FILE_VEILID_CORE_OPTIMIZED $PUBLIC_API_FILE_VEILID_CORE_OPTIMIZED + echo "Public API updated for 'veilid-core-optimized'." + fi if [ $veilid_remote_api_diff_status -eq 1 ]; then mv $PUBLIC_API_TMP_FILE_VEILID_REMOTE_API $PUBLIC_API_FILE_VEILID_REMOTE_API echo "Public API updated for 'veilid-remote-api'." diff --git a/veilid-core-optimized/Cargo.toml b/veilid-core-optimized/Cargo.toml new file mode 100644 index 00000000..7184b489 --- /dev/null +++ b/veilid-core-optimized/Cargo.toml @@ -0,0 +1,103 @@ +[package] +# --- Bumpversion match - do not reorder +name = "veilid-core-optimized" +version = "0.5.7" +# --- +description = "Internal crate for veilid-core, do not use directly. Monomorphic entry points for the hot dependencies of veilid-core, compiled at full optimization in every build profile" +repository.workspace = true +authors.workspace = true +license.workspace = true +edition.workspace = true +rust-version.workspace = true +documentation = "https://docs.rs/veilid-core-optimized" +homepage = "https://veilid.gitlab.io/developer-book/" + +[lib] +crate-type = ["rlib"] +path = "src/lib.rs" + +[features] +default = [] + +# One feature per cryptosystem; each pulls in only the crates that kind needs +enable-crypto-vld0 = [ + "dep:ed25519-dalek", + "dep:x25519-dalek", + "dep:curve25519-dalek", + "dep:blake3", + "dep:chacha20poly1305", + "dep:chacha20", + "dep:argon2", + "dep:blake2", + "dep:hpke", + "dep:rand_core", +] +enable-crypto-none = [ + "dep:blake3", + "dep:argon2", + "dep:blake2", + "dep:hpke", + "dep:subtle", + "dep:rand_core", +] + +[dependencies] +# Crypto +ed25519-dalek = { version = "3.0.0", optional = true, default-features = false, features = [ + "alloc", + "rand_core", + "digest", + "zeroize", +] } +x25519-dalek = { version = "3.0.0", optional = true, default-features = false, features = [ + "static_secrets", + "zeroize", + "precomputed-tables", +] } +curve25519-dalek = { version = "5.0.0", optional = true, default-features = false, features = [ + "alloc", + "zeroize", + "precomputed-tables", +] } +blake3 = { version = "1.8.3", optional = true } +blake2 = { version = "0.11.0-rc.5", optional = true, default-features = false, features = [ + "size_opt", +] } # Size optimization for c crate (via feature unification) +chacha20poly1305 = { version = "0.11.0", optional = true, features = [ + "zeroize", + "rand_core", +] } +chacha20 = { version = "0.10", optional = true, features = ["xchacha"] } +argon2 = { version = "0.6.0-rc.8", optional = true, default-features = false, features = [ + "alloc", + "password-hash", + "getrandom", + "rand_core", +] } +hpke = { version = "0.14.0", optional = true, default-features = false, features = [ + "alloc", + "x25519", + "chacha", +] } +subtle = { version = "2.6", optional = true, default-features = false } +rand_core = { version = "0.10.1", optional = true, default-features = false } + +# Compression +lz4_flex = { version = "0.14.0", default-features = false, features = [ + "alloc", + "safe-encode", + "safe-decode", +] } + +### Dependencies for WASM builds only + +[target.'cfg(all(target_arch = "wasm32", target_os = "unknown"))'.dependencies] +# argon2's getrandom needs the browser backend selected +getrandom = { version = "0.4.2", features = ["wasm_js"] } + +[package.metadata.cargo-machete] +# Feature unification only: blake2's size_opt for argon2, curve25519-dalek's tables for the dalek crates +ignored = ["blake2", "curve25519-dalek"] + +[lints] +workspace = true diff --git a/veilid-core-optimized/README.md b/veilid-core-optimized/README.md new file mode 100644 index 00000000..20a37f93 --- /dev/null +++ b/veilid-core-optimized/README.md @@ -0,0 +1,19 @@ + + + +Internal crate for veilid-core, do not use directly. + +Monomorphic entry points for the hot dependencies of veilid-core. Internal to veilid-core: +applications use the `veilid-core` API, never this crate directly. + +A Cargo profile override optimizes a dependency's own code, but generic and `#[inline]` +code is compiled in the crate that instantiates it, at that crate's opt-level. Every +function this crate exports is monomorphic and takes no caller-supplied type, so the +cipher, curve, hash, KDF, HPKE and LZ4 code it reaches is instantiated here, where the +workspace profiles keep it at `opt-level = 3` in every build mode. veilid-core can stay +size-optimized for the WASM blob and still run its crypto at full speed. + +Randomness comes in as a fill callback rather than an RNG type, so no `rand_core` version +binds this crate to its callers. + + diff --git a/veilid-core-optimized/public-api.txt b/veilid-core-optimized/public-api.txt new file mode 100644 index 00000000..741fa501 --- /dev/null +++ b/veilid-core-optimized/public-api.txt @@ -0,0 +1,73 @@ +pub mod veilid_core_optimized +pub enum veilid_core_optimized::Error +pub veilid_core_optimized::Error::Aead +pub veilid_core_optimized::Error::Argon2(alloc::string::String) +pub veilid_core_optimized::Error::Decompress(alloc::string::String) +pub veilid_core_optimized::Error::Hpke(alloc::string::String) +pub veilid_core_optimized::Error::InvalidKey +pub veilid_core_optimized::Error::InvalidLength +pub veilid_core_optimized::Error::InvalidLength::actual: usize +pub veilid_core_optimized::Error::InvalidLength::expected: usize +pub veilid_core_optimized::Error::InvalidPasswordHash(alloc::string::String) +pub veilid_core_optimized::Error::LengthMismatch +pub veilid_core_optimized::Error::NotContributory +pub veilid_core_optimized::Error::Signature(alloc::string::String) +impl core::error::Error for veilid_core_optimized::Error +impl core::fmt::Display for veilid_core_optimized::Error +pub fn veilid_core_optimized::Error::fmt(&self, &mut core::fmt::Formatter<'_>) -> core::fmt::Result +pub struct veilid_core_optimized::Blake3Hasher(_) +impl veilid_core_optimized::Blake3Hasher +pub fn veilid_core_optimized::Blake3Hasher::finalize(&self) -> [u8; 32] +pub fn veilid_core_optimized::Blake3Hasher::new() -> Self +pub fn veilid_core_optimized::Blake3Hasher::update(&mut self, &[u8]) +impl core::io::write::Write for veilid_core_optimized::Blake3Hasher +pub fn veilid_core_optimized::Blake3Hasher::flush(&mut self) -> core::io::error::Result<()> +pub fn veilid_core_optimized::Blake3Hasher::write(&mut self, &[u8]) -> core::io::error::Result +pub const veilid_core_optimized::ARGON2_SALT_MAX_LENGTH: usize +pub const veilid_core_optimized::ARGON2_SALT_MIN_LENGTH: usize +pub const veilid_core_optimized::BLAKE3_HASH_LENGTH: usize +pub const veilid_core_optimized::ED25519_PUBLIC_KEY_LENGTH: usize +pub const veilid_core_optimized::ED25519_SECRET_KEY_LENGTH: usize +pub const veilid_core_optimized::ED25519_SIGNATURE_LENGTH: usize +pub const veilid_core_optimized::HPKE_NONE_KEY_LENGTH: usize +pub const veilid_core_optimized::HPKE_X25519_KEY_LENGTH: usize +pub const veilid_core_optimized::X25519_KEY_LENGTH: usize +pub const veilid_core_optimized::XCHACHA20POLY1305_TAG_LENGTH: usize +pub const veilid_core_optimized::XCHACHA20_KEY_LENGTH: usize +pub const veilid_core_optimized::XCHACHA20_NONCE_LENGTH: usize +pub trait veilid_core_optimized::AeadBuffer: core::convert::AsRef<[u8]> + core::convert::AsMut<[u8]> +pub fn veilid_core_optimized::AeadBuffer::extend_from_slice(&mut self, &[u8]) +pub fn veilid_core_optimized::AeadBuffer::is_empty(&self) -> bool +pub fn veilid_core_optimized::AeadBuffer::len(&self) -> usize +pub fn veilid_core_optimized::AeadBuffer::truncate(&mut self, usize) +impl veilid_core_optimized::AeadBuffer for alloc::vec::Vec +pub fn alloc::vec::Vec::extend_from_slice(&mut self, &[u8]) +pub fn alloc::vec::Vec::is_empty(&self) -> bool +pub fn alloc::vec::Vec::len(&self) -> usize +pub fn alloc::vec::Vec::truncate(&mut self, usize) +pub fn veilid_core_optimized::argon2_derive_key(&[u8], &[u8], &mut [u8]) -> core::result::Result<(), veilid_core_optimized::Error> +pub fn veilid_core_optimized::argon2_hash_password(&[u8], &[u8]) -> core::result::Result +pub fn veilid_core_optimized::argon2_verify_password(&[u8], &str) -> core::result::Result +pub fn veilid_core_optimized::blake3_hash(&[u8]) -> [u8; 32] +pub fn veilid_core_optimized::blake3_hash_xof(&[u8], &mut [u8]) +pub fn veilid_core_optimized::ed25519_generate_keypair(veilid_core_optimized::FillBytes<'_>) -> ([u8; 32], [u8; 32]) +pub fn veilid_core_optimized::ed25519_public_to_x25519(&[u8]) -> core::result::Result<[u8; 32], veilid_core_optimized::Error> +pub fn veilid_core_optimized::ed25519_secret_to_x25519(&[u8]) -> core::result::Result<[u8; 32], veilid_core_optimized::Error> +pub fn veilid_core_optimized::ed25519_sign_prehashed(&[u8], &[u8], &[u8], &[u8]) -> core::result::Result<[u8; 64], veilid_core_optimized::Error> +pub fn veilid_core_optimized::ed25519_verify_prehashed(&[u8], &[u8], &[u8], &[u8]) -> core::result::Result +pub fn veilid_core_optimized::hpke_none_open(&[u8], &[u8], &[u8], &[u8], &[u8]) -> core::result::Result, veilid_core_optimized::Error> +pub fn veilid_core_optimized::hpke_none_seal(&[u8], &[u8], &[u8], &[u8], veilid_core_optimized::FillBytes<'_>) -> core::result::Result<(alloc::vec::Vec, alloc::vec::Vec), veilid_core_optimized::Error> +pub fn veilid_core_optimized::hpke_x25519_derive_keypair(&[u8]) -> ([u8; 32], [u8; 32]) +pub fn veilid_core_optimized::hpke_x25519_generate_keypair(veilid_core_optimized::FillBytes<'_>) -> ([u8; 32], [u8; 32]) +pub fn veilid_core_optimized::hpke_x25519_open(&[u8], &[u8], &[u8], &[u8], &[u8]) -> core::result::Result, veilid_core_optimized::Error> +pub fn veilid_core_optimized::hpke_x25519_public_from_secret(&[u8]) -> core::result::Result<[u8; 32], veilid_core_optimized::Error> +pub fn veilid_core_optimized::hpke_x25519_seal(&[u8], &[u8], &[u8], &[u8], veilid_core_optimized::FillBytes<'_>) -> core::result::Result<(alloc::vec::Vec, alloc::vec::Vec), veilid_core_optimized::Error> +pub fn veilid_core_optimized::lz4_compress_prepend_size(&[u8]) -> alloc::vec::Vec +pub fn veilid_core_optimized::lz4_decompress(&[u8], usize) -> core::result::Result, veilid_core_optimized::Error> +pub fn veilid_core_optimized::lz4_uncompressed_size(&[u8]) -> core::result::Result<(usize, &[u8]), veilid_core_optimized::Error> +pub fn veilid_core_optimized::x25519_diffie_hellman(&[u8], &[u8]) -> core::result::Result<[u8; 32], veilid_core_optimized::Error> +pub fn veilid_core_optimized::xchacha20_apply_keystream(&[u8], &[u8], &mut [u8]) -> core::result::Result<(), veilid_core_optimized::Error> +pub fn veilid_core_optimized::xchacha20_apply_keystream_b2b(&[u8], &[u8], &[u8], &mut [u8]) -> core::result::Result<(), veilid_core_optimized::Error> +pub fn veilid_core_optimized::xchacha20poly1305_decrypt_in_place(&[u8], &[u8], &[u8], &mut dyn veilid_core_optimized::AeadBuffer) -> core::result::Result<(), veilid_core_optimized::Error> +pub fn veilid_core_optimized::xchacha20poly1305_encrypt_in_place(&[u8], &[u8], &[u8], &mut dyn veilid_core_optimized::AeadBuffer) -> core::result::Result<(), veilid_core_optimized::Error> +pub type veilid_core_optimized::FillBytes<'a> = &'a mut dyn core::ops::function::FnMut(&mut [u8]) diff --git a/veilid-core-optimized/src/buffer.rs b/veilid-core-optimized/src/buffer.rs new file mode 100644 index 00000000..38ca9fe4 --- /dev/null +++ b/veilid-core-optimized/src/buffer.rs @@ -0,0 +1,35 @@ +/// A growable byte buffer an AEAD seals or opens in place: sealing appends the tag and +/// opening truncates it off. +pub trait AeadBuffer: AsRef<[u8]> + AsMut<[u8]> { + /// Bytes held + fn len(&self) -> usize; + + /// Whether nothing is held + fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Append `other` + fn extend_from_slice(&mut self, other: &[u8]); + + /// Drop everything past `len` + fn truncate(&mut self, len: usize); +} + +impl AeadBuffer for Vec { + fn len(&self) -> usize { + Vec::::len(self) + } + + fn is_empty(&self) -> bool { + Vec::::is_empty(self) + } + + fn extend_from_slice(&mut self, other: &[u8]) { + Vec::::extend_from_slice(self, other); + } + + fn truncate(&mut self, len: usize) { + Vec::::truncate(self, len); + } +} diff --git a/veilid-core-optimized/src/cipher.rs b/veilid-core-optimized/src/cipher.rs new file mode 100644 index 00000000..db43756a --- /dev/null +++ b/veilid-core-optimized/src/cipher.rs @@ -0,0 +1,111 @@ +use crate::{AeadBuffer, Error}; +use chacha20::cipher::{KeyIvInit, StreamCipher}; +use chacha20::XChaCha20; +use chacha20poly1305::{aead::AeadInOut as _, KeyInit as _, XChaCha20Poly1305}; + +/// Length of an XChaCha20 key in bytes +pub const XCHACHA20_KEY_LENGTH: usize = 32; +/// Length of an XChaCha20 nonce in bytes +pub const XCHACHA20_NONCE_LENGTH: usize = 24; +/// Bytes the Poly1305 tag adds to a sealed buffer +pub const XCHACHA20POLY1305_TAG_LENGTH: usize = 16; + +fn sized(bytes: &[u8]) -> Result<[u8; N], Error> { + bytes.try_into().map_err(|_| Error::InvalidLength { + expected: N, + actual: bytes.len(), + }) +} + +type KeyNonce = (chacha20poly1305::Key, chacha20poly1305::XNonce); + +fn key_nonce(key: &[u8], nonce: &[u8]) -> Result { + Ok(( + chacha20poly1305::Key::from(sized::(key)?), + chacha20poly1305::XNonce::from(sized::(nonce)?), + )) +} + +/// XOR `buf` with the XChaCha20 keystream for `key` and `nonce` +pub fn xchacha20_apply_keystream(key: &[u8], nonce: &[u8], buf: &mut [u8]) -> Result<(), Error> { + let (key, nonce) = key_nonce(key, nonce)?; + XChaCha20::new(&key, &nonce).apply_keystream(buf); + Ok(()) +} + +/// Write `input` XOR the XChaCha20 keystream for `key` and `nonce` into `output`, which +/// must be the same length +pub fn xchacha20_apply_keystream_b2b( + key: &[u8], + nonce: &[u8], + input: &[u8], + output: &mut [u8], +) -> Result<(), Error> { + if input.len() != output.len() { + return Err(Error::LengthMismatch); + } + let (key, nonce) = key_nonce(key, nonce)?; + XChaCha20::new(&key, &nonce).apply_keystream_b2b(input, output); + Ok(()) +} + +/// Seal `buffer` in place with XChaCha20-Poly1305, appending the tag +pub fn xchacha20poly1305_encrypt_in_place( + key: &[u8], + nonce: &[u8], + associated_data: &[u8], + buffer: &mut dyn AeadBuffer, +) -> Result<(), Error> { + let (key, nonce) = key_nonce(key, nonce)?; + XChaCha20Poly1305::new(&key) + .encrypt_in_place(&nonce, associated_data, &mut Adapter(buffer)) + .map_err(|_| Error::Aead) +} + +/// Open `buffer` in place with XChaCha20-Poly1305, removing the tag. `Aead` when the tag +/// does not authenticate. +pub fn xchacha20poly1305_decrypt_in_place( + key: &[u8], + nonce: &[u8], + associated_data: &[u8], + buffer: &mut dyn AeadBuffer, +) -> Result<(), Error> { + let (key, nonce) = key_nonce(key, nonce)?; + XChaCha20Poly1305::new(&key) + .decrypt_in_place(&nonce, associated_data, &mut Adapter(buffer)) + .map_err(|_| Error::Aead) +} + +// The AEAD's growable-buffer trait, over the caller's object-safe one +struct Adapter<'a>(&'a mut dyn AeadBuffer); + +impl AsRef<[u8]> for Adapter<'_> { + fn as_ref(&self) -> &[u8] { + self.0.as_ref() + } +} + +impl AsMut<[u8]> for Adapter<'_> { + fn as_mut(&mut self) -> &mut [u8] { + self.0.as_mut() + } +} + +impl chacha20poly1305::aead::Buffer for Adapter<'_> { + fn len(&self) -> usize { + self.0.len() + } + + fn is_empty(&self) -> bool { + self.0.is_empty() + } + + fn extend_from_slice(&mut self, other: &[u8]) -> chacha20poly1305::aead::Result<()> { + self.0.extend_from_slice(other); + Ok(()) + } + + fn truncate(&mut self, len: usize) { + self.0.truncate(len); + } +} diff --git a/veilid-core-optimized/src/compress.rs b/veilid-core-optimized/src/compress.rs new file mode 100644 index 00000000..b9fddd7b --- /dev/null +++ b/veilid-core-optimized/src/compress.rs @@ -0,0 +1,18 @@ +use crate::Error; +use lz4_flex::block; + +/// LZ4 block of `input`, prefixed with its uncompressed size +#[must_use] +pub fn lz4_compress_prepend_size(input: &[u8]) -> Vec { + block::compress_prepend_size(input) +} + +/// The uncompressed size a size-prefixed LZ4 block declares, and the block after the prefix +pub fn lz4_uncompressed_size(input: &[u8]) -> Result<(usize, &[u8]), Error> { + block::uncompressed_size(input).map_err(|e| Error::Decompress(e.to_string())) +} + +/// Decode an LZ4 block of known `uncompressed_size` +pub fn lz4_decompress(input: &[u8], uncompressed_size: usize) -> Result, Error> { + block::decompress(input, uncompressed_size).map_err(|e| Error::Decompress(e.to_string())) +} diff --git a/veilid-core-optimized/src/dh.rs b/veilid-core-optimized/src/dh.rs new file mode 100644 index 00000000..316f1c56 --- /dev/null +++ b/veilid-core-optimized/src/dh.rs @@ -0,0 +1,25 @@ +use crate::Error; +use x25519_dalek::{PublicKey, StaticSecret}; + +/// Length of an x25519 key in bytes +pub const X25519_KEY_LENGTH: usize = 32; + +fn key(bytes: &[u8]) -> Result<[u8; X25519_KEY_LENGTH], Error> { + bytes.try_into().map_err(|_| Error::InvalidLength { + expected: X25519_KEY_LENGTH, + actual: bytes.len(), + }) +} + +/// The x25519 shared secret of `secret` and `public`. `NotContributory` when the result is +/// the low-order point. +pub fn x25519_diffie_hellman( + secret: &[u8], + public: &[u8], +) -> Result<[u8; X25519_KEY_LENGTH], Error> { + let shared = StaticSecret::from(key(secret)?).diffie_hellman(&PublicKey::from(key(public)?)); + if !shared.was_contributory() { + return Err(Error::NotContributory); + } + Ok(shared.to_bytes()) +} diff --git a/veilid-core-optimized/src/error.rs b/veilid-core-optimized/src/error.rs new file mode 100644 index 00000000..862239f1 --- /dev/null +++ b/veilid-core-optimized/src/error.rs @@ -0,0 +1,53 @@ +use core::fmt; + +/// Why an operation refused or failed. Callers map these onto their own error types; the +/// string variants carry the underlying crate's message. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Error { + /// An input has the wrong length + InvalidLength { + /// Bytes required + expected: usize, + /// Bytes given + actual: usize, + }, + /// A key is not valid for its algorithm, or a keypair's halves do not match + InvalidKey, + /// Two buffers that must match in length do not + LengthMismatch, + /// Diffie-Hellman landed on a low-order point + NotContributory, + /// Signing failed + Signature(String), + /// AEAD authentication failed + Aead, + /// HPKE seal or open failed + Hpke(String), + /// Argon2 refused its parameters or failed + Argon2(String), + /// A password hash string is not in PHC format + InvalidPasswordHash(String), + /// An LZ4 block could not be decoded + Decompress(String), +} + +impl fmt::Display for Error { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::InvalidLength { expected, actual } => { + write!(f, "invalid length: expected {} got {}", expected, actual) + } + Self::InvalidKey => write!(f, "invalid key"), + Self::LengthMismatch => write!(f, "buffer lengths differ"), + Self::NotContributory => write!(f, "dh was not contributory"), + Self::Signature(e) => write!(f, "signing failed: {}", e), + Self::Aead => write!(f, "aead error"), + Self::Hpke(e) => write!(f, "hpke error: {}", e), + Self::Argon2(e) => write!(f, "argon2 error: {}", e), + Self::InvalidPasswordHash(e) => write!(f, "invalid password hash: {}", e), + Self::Decompress(e) => write!(f, "decompression failed: {}", e), + } + } +} + +impl std::error::Error for Error {} diff --git a/veilid-core-optimized/src/hash.rs b/veilid-core-optimized/src/hash.rs new file mode 100644 index 00000000..933e5efc --- /dev/null +++ b/veilid-core-optimized/src/hash.rs @@ -0,0 +1,50 @@ +/// Length of a BLAKE3 digest in bytes +pub const BLAKE3_HASH_LENGTH: usize = 32; + +/// BLAKE3 digest of `data` +#[must_use] +pub fn blake3_hash(data: &[u8]) -> [u8; BLAKE3_HASH_LENGTH] { + *blake3::hash(data).as_bytes() +} + +/// The first `out.len()` bytes of BLAKE3's extendable output over `data` +pub fn blake3_hash_xof(data: &[u8], out: &mut [u8]) { + let mut hasher = blake3::Hasher::new(); + hasher.update(data); + hasher.finalize_xof().fill(out); +} + +/// Incremental BLAKE3. Also a [`std::io::Write`] sink, so a reader hashes with +/// [`std::io::copy`]. +#[derive(Debug, Clone, Default)] +pub struct Blake3Hasher(blake3::Hasher); + +impl Blake3Hasher { + /// An empty hasher + #[must_use] + pub fn new() -> Self { + Self::default() + } + + /// Absorb `data` + pub fn update(&mut self, data: &[u8]) { + self.0.update(data); + } + + /// Digest of everything absorbed so far + #[must_use] + pub fn finalize(&self) -> [u8; BLAKE3_HASH_LENGTH] { + *self.0.finalize().as_bytes() + } +} + +impl std::io::Write for Blake3Hasher { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + self.0.update(buf); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} diff --git a/veilid-core-optimized/src/kdf.rs b/veilid-core-optimized/src/kdf.rs new file mode 100644 index 00000000..97f62dd3 --- /dev/null +++ b/veilid-core-optimized/src/kdf.rs @@ -0,0 +1,45 @@ +use crate::Error; +use argon2::{ + password_hash::{ + phc::{PasswordHash, Salt}, + PasswordHasher, PasswordVerifier, + }, + Argon2, +}; + +/// Shortest salt Argon2 accepts, in bytes +pub const ARGON2_SALT_MIN_LENGTH: usize = Salt::MIN_LENGTH; +/// Longest salt Argon2 accepts, in bytes +pub const ARGON2_SALT_MAX_LENGTH: usize = Salt::MAX_LENGTH; + +fn check_salt(salt: &[u8]) -> Result<(), Error> { + if salt.len() < ARGON2_SALT_MIN_LENGTH || salt.len() > ARGON2_SALT_MAX_LENGTH { + return Err(Error::Argon2("invalid salt length".to_owned())); + } + Ok(()) +} + +/// PHC string (`$argon2id$v=19$...`) of `password` hashed under `salt` with the default +/// Argon2id parameters +pub fn argon2_hash_password(password: &[u8], salt: &[u8]) -> Result { + check_salt(salt)?; + Argon2::default() + .hash_password_with_salt(password, salt) + .map(|h| h.to_string()) + .map_err(|e| Error::Argon2(e.to_string())) +} + +/// Whether `password` matches the PHC string `password_hash` +pub fn argon2_verify_password(password: &[u8], password_hash: &str) -> Result { + let parsed = + PasswordHash::new(password_hash).map_err(|e| Error::InvalidPasswordHash(e.to_string()))?; + Ok(Argon2::default().verify_password(password, &parsed).is_ok()) +} + +/// Fill `out` with Argon2id key material derived from `password` under `salt` +pub fn argon2_derive_key(password: &[u8], salt: &[u8], out: &mut [u8]) -> Result<(), Error> { + check_salt(salt)?; + Argon2::default() + .hash_password_into(password, salt, out) + .map_err(|e| Error::Argon2(e.to_string())) +} diff --git a/veilid-core-optimized/src/lib.rs b/veilid-core-optimized/src/lib.rs new file mode 100644 index 00000000..a31205e6 --- /dev/null +++ b/veilid-core-optimized/src/lib.rs @@ -0,0 +1,61 @@ +//! Internal crate for veilid-core, do not use directly. +//! +//! Monomorphic entry points for the hot dependencies of veilid-core. Internal to veilid-core: +//! applications use the `veilid-core` API, never this crate directly. +//! +//! A Cargo profile override optimizes a dependency's own code, but generic and `#[inline]` +//! code is compiled in the crate that instantiates it, at that crate's opt-level. Every +//! function this crate exports is monomorphic and takes no caller-supplied type, so the +//! cipher, curve, hash, KDF, HPKE and LZ4 code it reaches is instantiated here, where the +//! workspace profiles keep it at `opt-level = 3` in every build mode. veilid-core can stay +//! size-optimized for the WASM blob and still run its crypto at full speed. +//! +//! Randomness comes in as a fill callback rather than an RNG type, so no `rand_core` version +//! binds this crate to its callers. + +#![deny(warnings)] +#![warn(missing_docs)] + +mod buffer; +mod compress; +mod error; + +#[cfg(any(feature = "enable-crypto-vld0", feature = "enable-crypto-none"))] +mod hash; +#[cfg(any(feature = "enable-crypto-vld0", feature = "enable-crypto-none"))] +mod kdf; +#[cfg(any(feature = "enable-crypto-vld0", feature = "enable-crypto-none"))] +mod rng; +#[cfg(any(feature = "enable-crypto-vld0", feature = "enable-crypto-none"))] +mod seal; + +#[cfg(feature = "enable-crypto-vld0")] +mod cipher; +#[cfg(feature = "enable-crypto-vld0")] +mod dh; +#[cfg(feature = "enable-crypto-vld0")] +mod sign; + +#[cfg(feature = "enable-crypto-none")] +mod none_kem; + +pub use buffer::*; +pub use compress::*; +pub use error::*; + +#[cfg(any(feature = "enable-crypto-vld0", feature = "enable-crypto-none"))] +pub use hash::*; +#[cfg(any(feature = "enable-crypto-vld0", feature = "enable-crypto-none"))] +pub use kdf::*; +#[cfg(any(feature = "enable-crypto-vld0", feature = "enable-crypto-none"))] +pub use seal::*; + +#[cfg(feature = "enable-crypto-vld0")] +pub use cipher::*; +#[cfg(feature = "enable-crypto-vld0")] +pub use dh::*; +#[cfg(feature = "enable-crypto-vld0")] +pub use sign::*; + +/// The caller's source of cryptographically secure random bytes: fills the whole slice +pub type FillBytes<'a> = &'a mut dyn FnMut(&mut [u8]); diff --git a/veilid-core/src/crypto/crypto_system/none/none_kem.rs b/veilid-core-optimized/src/none_kem.rs similarity index 92% rename from veilid-core/src/crypto/crypto_system/none/none_kem.rs rename to veilid-core-optimized/src/none_kem.rs index 9d8c87e2..d243b2e0 100644 --- a/veilid-core/src/crypto/crypto_system/none/none_kem.rs +++ b/veilid-core-optimized/src/none_kem.rs @@ -1,11 +1,11 @@ -use ::hpke::hybrid_array::typenum::U32; -use ::hpke::hybrid_array::Array; -use ::hpke::kem::SharedSecret; -use ::hpke::rand_core::CryptoRng; -use ::hpke::{Deserializable, HpkeError, Kem as KemTrait, Serializable}; +use hpke::hybrid_array::typenum::U32; +use hpke::hybrid_array::Array; +use hpke::kem::SharedSecret; +use hpke::rand_core::CryptoRng; +use hpke::{Deserializable, HpkeError, Kem as KemTrait, Serializable}; use subtle::{Choice, ConstantTimeEq}; -const NONE_KEM_KEY_LENGTH: usize = 32; +pub(crate) const NONE_KEM_KEY_LENGTH: usize = 32; macro_rules! none_kem_key { ($name:ident) => { diff --git a/veilid-core-optimized/src/rng.rs b/veilid-core-optimized/src/rng.rs new file mode 100644 index 00000000..07df490a --- /dev/null +++ b/veilid-core-optimized/src/rng.rs @@ -0,0 +1,28 @@ +use crate::FillBytes; +use core::convert::Infallible; + +/// The caller's fill callback, seen by the dalek and RustCrypto crates as a CSPRNG +pub(crate) struct FillRng<'a>(pub(crate) FillBytes<'a>); + +impl rand_core::TryRng for FillRng<'_> { + type Error = Infallible; + + fn try_next_u32(&mut self) -> Result { + let mut b = [0u8; 4]; + (self.0)(&mut b); + Ok(u32::from_le_bytes(b)) + } + + fn try_next_u64(&mut self) -> Result { + let mut b = [0u8; 8]; + (self.0)(&mut b); + Ok(u64::from_le_bytes(b)) + } + + fn try_fill_bytes(&mut self, dst: &mut [u8]) -> Result<(), Self::Error> { + (self.0)(dst); + Ok(()) + } +} + +impl rand_core::TryCryptoRng for FillRng<'_> {} diff --git a/veilid-core-optimized/src/seal.rs b/veilid-core-optimized/src/seal.rs new file mode 100644 index 00000000..8e3b00cc --- /dev/null +++ b/veilid-core-optimized/src/seal.rs @@ -0,0 +1,154 @@ +//! HPKE (RFC 9180) base mode with HKDF-SHA256 and ChaCha20-Poly1305. The KEM is the only +//! thing that varies between cryptosystems, so each gets a pair of entry points. + +use crate::{rng::FillRng, Error, FillBytes}; +use hpke::{Deserializable, Kem as KemTrait, OpModeR, OpModeS, Serializable}; + +type Kdf = hpke::kdf::HkdfSha256; +type Aead = hpke::aead::ChaCha20Poly1305; + +// Instantiated here for every KEM, never in a caller +fn seal( + recipient: &[u8], + info: &[u8], + associated_data: &[u8], + plaintext: &[u8], + fill: FillBytes<'_>, +) -> Result<(Vec, Vec), Error> { + let recipient = Kem::PublicKey::from_bytes(recipient).map_err(|_| Error::InvalidKey)?; + let (enc, ciphertext) = hpke::single_shot_seal_with_rng::( + &OpModeS::Base, + &recipient, + info, + plaintext, + associated_data, + &mut FillRng(fill), + ) + .map_err(|e| Error::Hpke(e.to_string()))?; + Ok((enc.to_bytes().to_vec(), ciphertext)) +} + +fn open( + secret: &[u8], + enc: &[u8], + info: &[u8], + associated_data: &[u8], + ciphertext: &[u8], +) -> Result, Error> { + let secret = Kem::PrivateKey::from_bytes(secret).map_err(|_| Error::InvalidKey)?; + // enc comes from the blob; a bad one stays indistinguishable from a failed open + let enc = Kem::EncappedKey::from_bytes(enc).map_err(|e| Error::Hpke(e.to_string()))?; + hpke::single_shot_open::( + &OpModeR::Base, + &secret, + &enc, + info, + ciphertext, + associated_data, + ) + .map_err(|e| Error::Hpke(e.to_string())) +} + +#[cfg(feature = "enable-crypto-vld0")] +mod x25519 { + use super::*; + + type Kem = hpke::kem::X25519HkdfSha256; + + /// Length of a DHKEM(X25519) key, and of the `enc` a seal produces, in bytes + pub const HPKE_X25519_KEY_LENGTH: usize = 32; + + fn key_bytes(key: &impl Serializable) -> [u8; HPKE_X25519_KEY_LENGTH] { + let mut out = [0u8; HPKE_X25519_KEY_LENGTH]; + key.write_exact(&mut out); + out + } + + /// A fresh DHKEM(X25519) keypair as `(secret, public)`, via RFC 9180 DeriveKeyPair over + /// random input keying material + pub fn hpke_x25519_generate_keypair( + fill: FillBytes<'_>, + ) -> ([u8; HPKE_X25519_KEY_LENGTH], [u8; HPKE_X25519_KEY_LENGTH]) { + let (sk, pk) = Kem::gen_keypair_with_rng(&mut FillRng(fill)); + (key_bytes(&sk), key_bytes(&pk)) + } + + /// RFC 9180 DeriveKeyPair for DHKEM(X25519), as `(secret, public)` + #[must_use] + pub fn hpke_x25519_derive_keypair( + ikm: &[u8], + ) -> ([u8; HPKE_X25519_KEY_LENGTH], [u8; HPKE_X25519_KEY_LENGTH]) { + let (sk, pk) = Kem::derive_keypair(ikm); + (key_bytes(&sk), key_bytes(&pk)) + } + + /// The public key of a DHKEM(X25519) secret key + pub fn hpke_x25519_public_from_secret( + secret: &[u8], + ) -> Result<[u8; HPKE_X25519_KEY_LENGTH], Error> { + let sk = + ::PrivateKey::from_bytes(secret).map_err(|_| Error::InvalidKey)?; + Ok(key_bytes(&Kem::sk_to_pk(&sk))) + } + + /// Seal `plaintext` to `recipient`, returning `(enc, ciphertext)`. `InvalidKey` when + /// `recipient` is not a DHKEM(X25519) public key. + pub fn hpke_x25519_seal( + recipient: &[u8], + info: &[u8], + associated_data: &[u8], + plaintext: &[u8], + fill: FillBytes<'_>, + ) -> Result<(Vec, Vec), Error> { + seal::(recipient, info, associated_data, plaintext, fill) + } + + /// Open `ciphertext` sealed to `secret` with `enc`. `InvalidKey` when `secret` is not a + /// DHKEM(X25519) secret key; `Hpke` for anything wrong with the blob. + pub fn hpke_x25519_open( + secret: &[u8], + enc: &[u8], + info: &[u8], + associated_data: &[u8], + ciphertext: &[u8], + ) -> Result, Error> { + open::(secret, enc, info, associated_data, ciphertext) + } +} + +#[cfg(feature = "enable-crypto-vld0")] +pub use x25519::*; + +#[cfg(feature = "enable-crypto-none")] +mod none { + use super::*; + use crate::none_kem::NoneKem; + + /// Length of a NONE KEM key, and of the `enc` a seal produces, in bytes + pub const HPKE_NONE_KEY_LENGTH: usize = crate::none_kem::NONE_KEM_KEY_LENGTH; + + /// Seal `plaintext` to `recipient` under the NONE KEM, returning `(enc, ciphertext)` + pub fn hpke_none_seal( + recipient: &[u8], + info: &[u8], + associated_data: &[u8], + plaintext: &[u8], + fill: FillBytes<'_>, + ) -> Result<(Vec, Vec), Error> { + seal::(recipient, info, associated_data, plaintext, fill) + } + + /// Open `ciphertext` sealed to `secret` with `enc` under the NONE KEM + pub fn hpke_none_open( + secret: &[u8], + enc: &[u8], + info: &[u8], + associated_data: &[u8], + ciphertext: &[u8], + ) -> Result, Error> { + open::(secret, enc, info, associated_data, ciphertext) + } +} + +#[cfg(feature = "enable-crypto-none")] +pub use none::*; diff --git a/veilid-core-optimized/src/sign.rs b/veilid-core-optimized/src/sign.rs new file mode 100644 index 00000000..74614ae8 --- /dev/null +++ b/veilid-core-optimized/src/sign.rs @@ -0,0 +1,95 @@ +use crate::{rng::FillRng, Error, FillBytes}; +use ed25519_dalek::{Digest as _, Sha512, Signature, SigningKey, VerifyingKey}; + +/// Length of an ed25519 public key in bytes +pub const ED25519_PUBLIC_KEY_LENGTH: usize = 32; +/// Length of an ed25519 secret key in bytes +pub const ED25519_SECRET_KEY_LENGTH: usize = 32; +/// Length of an ed25519 signature in bytes +pub const ED25519_SIGNATURE_LENGTH: usize = 64; + +fn sized(bytes: &[u8]) -> Result<[u8; N], Error> { + bytes.try_into().map_err(|_| Error::InvalidLength { + expected: N, + actual: bytes.len(), + }) +} + +fn verifying_key(public: &[u8]) -> Result { + VerifyingKey::from_bytes(&sized::(public)?) + .map_err(|_| Error::InvalidKey) +} + +fn signing_key(public: &[u8], secret: &[u8]) -> Result { + let mut keypair = [0u8; ED25519_SECRET_KEY_LENGTH + ED25519_PUBLIC_KEY_LENGTH]; + keypair[..ED25519_SECRET_KEY_LENGTH] + .copy_from_slice(&sized::(secret)?); + keypair[ED25519_SECRET_KEY_LENGTH..] + .copy_from_slice(&sized::(public)?); + SigningKey::from_keypair_bytes(&keypair).map_err(|_| Error::InvalidKey) +} + +fn prehash(data: &[u8]) -> Sha512 { + let mut digest = Sha512::default(); + digest.update(data); + digest +} + +/// A fresh keypair as `(public, secret)` +pub fn ed25519_generate_keypair( + fill: FillBytes<'_>, +) -> ( + [u8; ED25519_PUBLIC_KEY_LENGTH], + [u8; ED25519_SECRET_KEY_LENGTH], +) { + let signing_key = SigningKey::generate(&mut FillRng(fill)); + ( + signing_key.verifying_key().to_bytes(), + signing_key.to_bytes(), + ) +} + +/// Ed25519ph signature over `data` with `domain` as the context string. `InvalidKey` when +/// `public` and `secret` are not the halves of one keypair. +pub fn ed25519_sign_prehashed( + public: &[u8], + secret: &[u8], + data: &[u8], + domain: &[u8], +) -> Result<[u8; ED25519_SIGNATURE_LENGTH], Error> { + signing_key(public, secret)? + .sign_prehashed(prehash(data), Some(domain)) + .map(|s| s.to_bytes()) + .map_err(|e| Error::Signature(e.to_string())) +} + +/// Strict Ed25519ph verification of `signature` over `data` under `domain`. `Ok(false)` +/// when the signature does not match; `InvalidKey` when `public` is not a curve point. +pub fn ed25519_verify_prehashed( + public: &[u8], + data: &[u8], + domain: &[u8], + signature: &[u8], +) -> Result { + let key = verifying_key(public)?; + let signature = Signature::from_bytes(&sized::(signature)?); + Ok(key + .verify_prehashed_strict(prehash(data), Some(domain), &signature) + .is_ok()) +} + +/// The x25519 public key on the same curve point as an ed25519 public key +pub fn ed25519_public_to_x25519(public: &[u8]) -> Result<[u8; ED25519_PUBLIC_KEY_LENGTH], Error> { + Ok(verifying_key(public)?.to_montgomery().to_bytes()) +} + +/// The x25519 secret key an ed25519 secret key expands to +pub fn ed25519_secret_to_x25519(secret: &[u8]) -> Result<[u8; ED25519_SECRET_KEY_LENGTH], Error> { + let secret = sized::(secret)?; + // The raw first half of the expanded key, not SigningKey::to_scalar(): that one reduces + // the scalar (dalek-cryptography/curve25519-dalek#565) + let hash: [u8; 64] = Sha512::default().chain_update(secret).finalize().into(); + let mut out = [0u8; ED25519_SECRET_KEY_LENGTH]; + out.copy_from_slice(&hash[..ED25519_SECRET_KEY_LENGTH]); + Ok(out) +} diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index c5967e88..a1b2fb7a 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -56,29 +56,9 @@ rt-async-std = ["async-std", "veilid-tools/rt-async-std"] rt-tokio = ["tokio", "veilid-tools/rt-tokio"] rt-wasm-bindgen = ["veilid-tools/rt-wasm-bindgen"] -# Crypto support features -enable-crypto-vld0 = [ - "ed25519-dalek", - "x25519-dalek", - "curve25519-dalek", - "blake3", - "chacha20poly1305", - "chacha20", - "argon2", - "blake2", # To apply optimization for argon2 code size - "hpke", -] -enable-crypto-none = [ - "ed25519-dalek", - "curve25519-dalek", - "blake3", - "argon2", - "digest", - "blake2", # To apply optimization for argon2 code size - "chacha20poly1305", - "hpke", - "subtle", -] +# Crypto support features; the primitives live in veilid-core-optimized +enable-crypto-vld0 = ["veilid-core-optimized/enable-crypto-vld0"] +enable-crypto-none = ["veilid-core-optimized/enable-crypto-none"] # Protocol support features enable-protocol-wss = ["webpki-roots"] @@ -153,46 +133,8 @@ futures-util = { version = "0.3.31", default-features = false, features = [ flume = { version = "0.12.0", features = ["async"] } arc-swap = { version = "1.8.0" } -# Crypto -ed25519-dalek = { version = "3.0.0", optional = true, default-features = false, features = [ - "alloc", - "rand_core", - "digest", - "zeroize", -] } -x25519-dalek = { version = "3.0.0", optional = true, default-features = false, features = [ - "static_secrets", - "zeroize", - "precomputed-tables", -] } -curve25519-dalek = { version = "5.0.0", optional = true, default-features = false, features = [ - "alloc", - "zeroize", - "precomputed-tables", -] } -blake3 = { version = "1.8.3", optional = true } -blake2 = { version = "0.11.0-rc.5", optional = true, default-features = false, features = [ - "size_opt", -] } # Size optimization for c crate (via feature unification) -chacha20poly1305 = { version = "0.11.0", optional = true, features = [ - "bytes", - "zeroize", - "rand_core", -] } -chacha20 = { version = "0.10", optional = true, features = ["xchacha"] } -argon2 = { version = "0.6.0-rc.8", optional = true, default-features = false, features = [ - "alloc", - "password-hash", - "getrandom", - "rand_core", -] } -digest = { version = "0.11", optional = true, features = ["rand_core"] } -hpke = { version = "0.14.0", optional = true, default-features = false, features = [ - "alloc", - "x25519", - "chacha", -] } -subtle = { version = "2.6", optional = true, default-features = false } +# Crypto and compression primitives, kept at full optimization in every profile +veilid-core-optimized = { version = "0.5.7", path = "../veilid-core-optimized", default-features = false } # Serialization capnp = { version = "0.26.1", default-features = false, features = [ @@ -205,10 +147,6 @@ serde_json = { version = "1.0.149" } data-encoding = { version = "2.8.0" } zeroize = { version = "1" } schemars = { version = "1.0.3", optional = true } -lz4_flex = { version = "0.13.1", default-features = false, features = [ - "safe-encode", - "safe-decode", -] } indent = "0.1.1" bytesize = { version = "2.3.1", optional = true } diff --git a/veilid-core/public-api.txt b/veilid-core/public-api.txt index f33927cb..cddd2c0b 100644 --- a/veilid-core/public-api.txt +++ b/veilid-core/public-api.txt @@ -1,5 +1,4 @@ pub mod veilid_core -pub use veilid_core::Buffer pub use veilid_core::async_trait pub use veilid_core::async_trait pub use veilid_core::bytes @@ -3346,6 +3345,8 @@ impl veilid_core::ToVeilidAPIErrorArgument for veilid_core::ValueSubkeyRangeSet pub fn veilid_core::ValueSubkeyRangeSet::to_veilid_api_error_argument(&self) -> alloc::string::String impl veilid_core::ToVeilidAPIErrorArgument for veilid_core::VeilidCapability pub fn veilid_core::VeilidCapability::to_veilid_api_error_argument(&self) -> alloc::string::String +impl veilid_core::ToVeilidAPIErrorArgument for veilid_core_optimized::error::Error +pub fn veilid_core_optimized::error::Error::to_veilid_api_error_argument(&self) -> alloc::string::String impl veilid_core::ToVeilidAPIErrorArgument for &T pub fn &T::to_veilid_api_error_argument(&self) -> alloc::string::String impl veilid_core::ToVeilidAPIErrorArgument for core::ops::range::Range diff --git a/veilid-core/smoketest/Cargo.lock b/veilid-core/smoketest/Cargo.lock index 596750e8..da30552e 100644 --- a/veilid-core/smoketest/Cargo.lock +++ b/veilid-core/smoketest/Cargo.lock @@ -23,7 +23,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ - "bytes", "crypto-common 0.2.2", "inout 0.2.2", ] @@ -1819,9 +1818,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" [[package]] name = "macro_rules_attribute" @@ -3192,25 +3191,18 @@ name = "veilid-core" version = "0.5.7" dependencies = [ "arc-swap", - "argon2", "async-std", "async-trait", "async-tungstenite", "backtrace", - "blake2", - "blake3", "bosion", "bytesize", "capnp", "capnpc", "cfg-if", - "chacha20", - "chacha20poly1305", "clap", - "curve25519-dalek", "data-encoding", "directories", - "ed25519-dalek", "enumset", "eyre", "flume", @@ -3220,7 +3212,6 @@ dependencies = [ "glob", "gloo-events", "hex", - "hpke", "indent", "jni", "js-sys", @@ -3230,7 +3221,6 @@ dependencies = [ "keyvaluedb-web", "lazy_static", "libc", - "lz4_flex", "macro_rules_attribute", "num-traits", "pastey", @@ -3248,6 +3238,7 @@ dependencies = [ "tracing-subscriber", "tsify", "tsify-macros", + "veilid-core-optimized", "veilid-hashlink", "veilid-tools", "wasm-bindgen", @@ -3255,10 +3246,27 @@ dependencies = [ "weak-table", "web-sys", "ws_stream_wasm", - "x25519-dalek", "zeroize", ] +[[package]] +name = "veilid-core-optimized" +version = "0.5.7" +dependencies = [ + "argon2", + "blake2", + "blake3", + "chacha20", + "chacha20poly1305", + "curve25519-dalek", + "ed25519-dalek", + "getrandom 0.4.3", + "hpke", + "lz4_flex", + "rand_core 0.10.1", + "x25519-dalek", +] + [[package]] name = "veilid-hashlink" version = "0.1.3" diff --git a/veilid-core/src/crypto/crypto_system/buffer.rs b/veilid-core/src/crypto/crypto_system/buffer.rs index 590bca30..5153c69c 100644 --- a/veilid-core/src/crypto/crypto_system/buffer.rs +++ b/veilid-core/src/crypto/crypto_system/buffer.rs @@ -1,7 +1,5 @@ use super::*; -use chacha20poly1305 as ch; - /// A growable, mutable byte buffer that AEAD operations can encrypt or decrypt in place. /// /// Implemented for `Vec` and `BytesMut`. Encryption appends the authentication tag @@ -56,7 +54,7 @@ impl CryptoSystemBuffer for BytesMut { } } -/// Adapts a [`CryptoSystemBuffer`] to the `chacha20poly1305` AEAD `Buffer` trait so in-place AEAD +/// Adapts a [`CryptoSystemBuffer`] to the optimized crate's AEAD buffer trait so in-place AEAD /// operations can grow and truncate the underlying `Vec` or `BytesMut`. pub(super) struct BufferWrapper<'a> { buffer: &'a mut dyn CryptoSystemBuffer, @@ -78,16 +76,15 @@ impl<'a> AsMut<[u8]> for BufferWrapper<'a> { self.buffer.as_mut() } } -impl<'a> ch::aead::Buffer for BufferWrapper<'a> { +impl<'a> veilid_core_optimized::AeadBuffer for BufferWrapper<'a> { fn len(&self) -> usize { self.buffer.len() } fn is_empty(&self) -> bool { self.buffer.is_empty() } - fn extend_from_slice(&mut self, other: &[u8]) -> Result<(), ch::Error> { + fn extend_from_slice(&mut self, other: &[u8]) { self.buffer.extend_from_slice(other); - Ok(()) } fn truncate(&mut self, len: usize) { self.buffer.truncate(len) diff --git a/veilid-core/src/crypto/crypto_system/hpke.rs b/veilid-core/src/crypto/crypto_system/hpke.rs index 43693c86..edca7a6d 100644 --- a/veilid-core/src/crypto/crypto_system/hpke.rs +++ b/veilid-core/src/crypto/crypto_system/hpke.rs @@ -1,16 +1,10 @@ use super::*; -use ::hpke::{Deserializable, Kem as KemTrait, OpModeR, OpModeS, Serializable}; - // Wire and info constants, isolated because the HPKE RFC is still in review pub(crate) const HPKE_VERSION_1: u8 = 1; pub(crate) const HPKE_INFO_LABEL: &[u8] = b"veilid-hpke/1"; pub(crate) const HPKE_HEADER_LENGTH: usize = 5; -// One suite per kind; kinds supply only the Kem type parameter -type HpkeKdf = ::hpke::kdf::HkdfSha256; -type HpkeAead = ::hpke::aead::ChaCha20Poly1305; - pub(crate) fn hpke_info(kind: CryptoKind) -> Vec { let mut info = Vec::with_capacity(HPKE_INFO_LABEL.len() + 4); info.extend_from_slice(HPKE_INFO_LABEL); @@ -53,51 +47,3 @@ pub(crate) fn parse_sealed( } Ok(sealed[HPKE_HEADER_LENGTH..].split_at(enc_length)) } - -pub(crate) fn hpke_seal_core( - pk_bytes: &[u8], - info: &[u8], - aad: &[u8], - plaintext: &[u8], - csprng: &mut impl ::hpke::rand_core::CryptoRng, -) -> VeilidAPIResult<(Vec, Vec)> { - let pk = Kem::PublicKey::from_bytes(pk_bytes).map_err(|e| { - VeilidAPIError::invalid_argument("hpke_seal", "recipient", map_to_string(e)) - })?; - let (enc, ciphertext) = ::hpke::single_shot_seal_with_rng::( - &OpModeS::Base, - &pk, - info, - plaintext, - aad, - csprng, - ) - .map_err(map_to_string) - .map_err(VeilidAPIError::generic)?; - Ok((enc.to_bytes().to_vec(), ciphertext)) -} - -pub(crate) fn hpke_open_core( - sk_bytes: &[u8], - enc: &[u8], - info: &[u8], - aad: &[u8], - ciphertext: &[u8], -) -> VeilidAPIResult> { - let sk = Kem::PrivateKey::from_bytes(sk_bytes) - .map_err(|e| VeilidAPIError::invalid_argument("hpke_open", "secret", map_to_string(e)))?; - // enc comes from the blob; failures here stay indistinguishable from decrypt failure - let enc = Kem::EncappedKey::from_bytes(enc) - .map_err(map_to_string) - .map_err(VeilidAPIError::generic)?; - ::hpke::single_shot_open::( - &OpModeR::Base, - &sk, - &enc, - info, - ciphertext, - aad, - ) - .map_err(map_to_string) - .map_err(VeilidAPIError::generic) -} diff --git a/veilid-core/src/crypto/crypto_system/none/blake3digest512.rs b/veilid-core/src/crypto/crypto_system/none/blake3digest512.rs deleted file mode 100644 index 1b2a3026..00000000 --- a/veilid-core/src/crypto/crypto_system/none/blake3digest512.rs +++ /dev/null @@ -1,116 +0,0 @@ -use digest::array::{typenum::U64, Array}; -use digest::{Digest, FixedOutput, FixedOutputReset, Output, OutputSizeUser, Reset, Update}; - -/// A BLAKE3 hasher exposing a fixed 64-byte (512-bit) output via the RustCrypto `Digest` traits. -/// -/// BLAKE3 is natively an extendable-output function; this wraps it as a fixed-size digest by -/// reading 64 bytes from its XOF, so it can be used wherever a 512-bit `Digest` is expected. -pub struct Blake3Digest512 { - dig: blake3::Hasher, -} - -impl OutputSizeUser for Blake3Digest512 { - type OutputSize = U64; -} - -impl Update for Blake3Digest512 { - fn update(&mut self, data: &[u8]) { - self.dig.update(data); - } -} - -impl FixedOutput for Blake3Digest512 { - fn finalize_into(self, out: &mut Output) { - let mut b = [0u8; 64]; - self.dig.finalize_xof().fill(&mut b); - out[0..64].copy_from_slice(&b[0..64]); - } -} - -impl Reset for Blake3Digest512 { - fn reset(&mut self) { - self.dig.reset(); - } -} - -impl FixedOutputReset for Blake3Digest512 { - fn finalize_into_reset(&mut self, out: &mut Output) { - let mut b = [0u8; 64]; - self.dig.finalize_xof().fill(&mut b); - out[0..64].copy_from_slice(&b[0..64]); - self.dig.reset(); - } -} - -impl Digest for Blake3Digest512 { - fn new() -> Self { - Self { - dig: blake3::Hasher::new(), - } - } - - fn new_with_prefix(data: impl AsRef<[u8]>) -> Self { - Self::new().chain_update(data) - } - - fn chain_update(mut self, data: impl AsRef<[u8]>) -> Self - where - Self: Sized, - { - ::update(&mut self, data.as_ref()); - self - } - - fn finalize(self) -> Output { - let mut b = [0u8; 64]; - self.dig.finalize_xof().fill(&mut b); - let mut out = Array::::default(); - out[0..64].copy_from_slice(&b[0..64]); - out - } - - fn finalize_reset(&mut self) -> Output { - let mut b = [0u8; 64]; - self.dig.finalize_xof().fill(&mut b); - let mut out = Array::::default(); - out[0..64].copy_from_slice(&b[0..64]); - self.dig.reset(); - out - } - - fn output_size() -> usize { - 64 - } - - fn digest(data: impl AsRef<[u8]>) -> Output { - let mut dig = blake3::Hasher::new(); - dig.update(data.as_ref()); - let mut b = [0u8; 64]; - dig.finalize_xof().fill(&mut b); - let mut out = Array::::default(); - out[0..64].copy_from_slice(&b[0..64]); - out - } - - fn update(&mut self, data: impl AsRef<[u8]>) { - ::update(self, data.as_ref()) - } - - fn finalize_into(self, out: &mut Output) { - ::finalize_into(self, out) - } - - fn finalize_into_reset(&mut self, out: &mut Output) - where - Self: FixedOutputReset, - { - ::finalize_into_reset(self, out) - } - - fn reset(&mut self) - where - Self: Reset, - { - ::reset(self); - } -} diff --git a/veilid-core/src/crypto/crypto_system/none/mod.rs b/veilid-core/src/crypto/crypto_system/none/mod.rs index a21267bb..ff00e715 100644 --- a/veilid-core/src/crypto/crypto_system/none/mod.rs +++ b/veilid-core/src/crypto/crypto_system/none/mod.rs @@ -1,16 +1,11 @@ -mod blake3digest512; -mod none_kem; mod sizes; use super::*; -use super::hpke::{build_sealed, hpke_info, hpke_open_core, hpke_seal_core, parse_sealed}; -use argon2::password_hash::phc::Salt; -use blake3digest512::Blake3Digest512; +use super::hpke::{build_sealed, hpke_info, parse_sealed}; use data_encoding::BASE64URL_NOPAD; -use digest::rand_core::Rng as _; -use digest::Digest; -use none_kem::NoneKem; +use veilid_core_optimized as opt; + const NONE_AEAD_OVERHEAD: usize = NONE_PUBLIC_KEY_LENGTH; /// [`CryptoKind`] fourcc identifying the NONE cryptosystem. pub const CRYPTO_KIND_NONE: CryptoKind = CryptoKind::new(*b"NONE"); @@ -19,10 +14,9 @@ pub const CRYPTO_KIND_NONE_FOURCC: u32 = u32::from_be_bytes(*b"NONE"); pub use sizes::*; pub(crate) fn none_generate_keypair() -> KeyPair { - let mut csprng = VeilidRng {}; let mut pub_bytes = [0u8; NONE_PUBLIC_KEY_LENGTH]; let mut sec_bytes = [0u8; NONE_SECRET_KEY_LENGTH]; - csprng.fill_bytes(&mut pub_bytes); + random_bytes(&mut pub_bytes); for n in 0..NONE_PUBLIC_KEY_LENGTH { sec_bytes[n] = !pub_bytes[n]; } @@ -32,10 +26,9 @@ pub(crate) fn none_generate_keypair() -> KeyPair { } pub(crate) fn none_generate_kem_keypair() -> KemKeyPair { - let mut csprng = VeilidRng {}; let mut ek_bytes = [0u8; NONE_ENCAPSULATION_KEY_LENGTH]; let mut dk_bytes = [0u8; NONE_DECAPSULATION_KEY_LENGTH]; - csprng.fill_bytes(&mut ek_bytes); + random_bytes(&mut ek_bytes); for n in 0..NONE_ENCAPSULATION_KEY_LENGTH { dk_bytes[n] = !ek_bytes[n]; } @@ -94,9 +87,9 @@ fn do_aead_keyblob( let mut blob = nonce.to_vec(); blob.extend_from_slice(&[0u8; 8]); let mut blob = do_xor_32(&blob, shared_secret.ref_value())?; - let ad_hash = blake3::hash(associated_data.unwrap_or(b"")); + let ad_hash = opt::blake3_hash(associated_data.unwrap_or(b"")); for n in 0..32 { - blob[n] ^= ad_hash.as_bytes()[n]; + blob[n] ^= ad_hash[n]; } Ok(blob) } @@ -114,6 +107,13 @@ fn is_bytes_eq_32(a: &[u8], v: u8) -> VeilidAPIResult { Ok(true) } +// The signature's message digest: 64 bytes of BLAKE3 extendable output +fn signature_digest(data: &[u8]) -> [u8; NONE_SIGNATURE_LENGTH] { + let mut digest = [0u8; NONE_SIGNATURE_LENGTH]; + opt::blake3_hash_xof(data, &mut digest); + digest +} + /// None CryptoSystem pub(crate) struct CryptoSystemNONE { registry: VeilidComponentRegistry, @@ -149,7 +149,7 @@ impl CryptoSystem for CryptoSystemNONE { bytes } fn hash_password(&self, password: &[u8], salt: &[u8]) -> VeilidAPIResult { - if salt.len() < Salt::MIN_LENGTH || salt.len() > Salt::MAX_LENGTH { + if salt.len() < opt::ARGON2_SALT_MIN_LENGTH || salt.len() > opt::ARGON2_SALT_MAX_LENGTH { apibail_generic!("invalid salt length"); } Ok(format!( @@ -169,14 +169,14 @@ impl CryptoSystem for CryptoSystemNONE { } fn derive_shared_secret(&self, password: &[u8], salt: &[u8]) -> VeilidAPIResult { - if salt.len() < Salt::MIN_LENGTH || salt.len() > Salt::MAX_LENGTH { + if salt.len() < opt::ARGON2_SALT_MIN_LENGTH || salt.len() > opt::ARGON2_SALT_MAX_LENGTH { apibail_generic!("invalid salt length"); } Ok(SharedSecret::new( CRYPTO_KIND_NONE, - BareSharedSecret::new( - blake3::hash(self.hash_password(password, salt)?.as_bytes()).as_bytes(), - ), + BareSharedSecret::new(&opt::blake3_hash( + self.hash_password(password, salt)?.as_bytes(), + )), )) } @@ -208,13 +208,17 @@ impl CryptoSystem for CryptoSystemNONE { aad: &[u8], plaintext: &[u8], ) -> VeilidAPIResult> { - let (enc, ciphertext) = hpke_seal_core::( + let (enc, ciphertext) = opt::hpke_none_seal( recipient.ref_value(), &hpke_info(CRYPTO_KIND_NONE), aad, plaintext, - &mut VeilidRng {}, - )?; + &mut random_bytes, + ) + .map_err(|e| match e { + opt::Error::InvalidKey => VeilidAPIError::invalid_argument("hpke_seal", "recipient", e), + e => VeilidAPIError::generic(e), + })?; Ok(build_sealed(CRYPTO_KIND_NONE, &enc, &ciphertext)) } @@ -225,13 +229,17 @@ impl CryptoSystem for CryptoSystemNONE { sealed: &[u8], ) -> VeilidAPIResult> { let (enc, ciphertext) = parse_sealed(CRYPTO_KIND_NONE, NONE_HPKE_ENC_LENGTH, sealed)?; - hpke_open_core::( + opt::hpke_none_open( secret.ref_value(), enc, &hpke_info(CRYPTO_KIND_NONE), aad, ciphertext, ) + .map_err(|e| match e { + opt::Error::InvalidKey => VeilidAPIError::invalid_argument("hpke_open", "secret", e), + e => VeilidAPIError::generic(e), + }) } fn generate_keypair(&self) -> KeyPair { @@ -279,15 +287,15 @@ impl CryptoSystem for CryptoSystemNONE { fn generate_hash(&self, data: &[u8]) -> HashDigest { HashDigest::new( CRYPTO_KIND_NONE, - BareHashDigest::new(blake3::hash(data).as_bytes()), + BareHashDigest::new(&opt::blake3_hash(data)), ) } fn generate_hash_reader(&self, reader: &mut dyn std::io::Read) -> VeilidAPIResult { - let mut hasher = blake3::Hasher::new(); + let mut hasher = opt::Blake3Hasher::new(); std::io::copy(reader, &mut hasher).map_err(|e| VeilidAPIError::generic(e.to_string()))?; Ok(PublicKey::new( CRYPTO_KIND_NONE, - BarePublicKey::new(hasher.finalize().as_bytes()), + BarePublicKey::new(&hasher.finalize()), )) } @@ -342,9 +350,8 @@ impl CryptoSystem for CryptoSystemNONE { } fn validate_hash(&self, data: &[u8], hash_digest: &HashDigest) -> VeilidAPIResult { self.check_hash_digest(hash_digest)?; - let out_hash = blake3::hash(data); - let bytes = out_hash.as_bytes(); - Ok(*bytes == **hash_digest.ref_value()) + let bytes = opt::blake3_hash(data); + Ok(bytes == **hash_digest.ref_value()) } fn validate_hash_reader( &self, @@ -352,11 +359,10 @@ impl CryptoSystem for CryptoSystemNONE { hash_digest: &HashDigest, ) -> VeilidAPIResult { self.check_hash_digest(hash_digest)?; - let mut hasher = blake3::Hasher::new(); + let mut hasher = opt::Blake3Hasher::new(); std::io::copy(reader, &mut hasher).map_err(|e| VeilidAPIError::generic(e.to_string()))?; - let out_hash = hasher.finalize(); - let bytes = out_hash.as_bytes(); - Ok(*bytes == **hash_digest.ref_value()) + let bytes = hasher.finalize(); + Ok(bytes == **hash_digest.ref_value()) } // Authentication @@ -379,10 +385,7 @@ impl CryptoSystem for CryptoSystemNONE { )); } - let mut dig = Blake3Digest512::new(); - dig.update(data); - let sig = dig.finalize(); - let in_sig_bytes: [u8; NONE_SIGNATURE_LENGTH] = sig.into(); + let in_sig_bytes = signature_digest(data); let mut sig_bytes = [0u8; NONE_SIGNATURE_LENGTH]; sig_bytes[0..32].copy_from_slice(&in_sig_bytes[0..32]); sig_bytes[32..64] @@ -402,12 +405,9 @@ impl CryptoSystem for CryptoSystemNONE { self.check_public_key(public_key)?; self.check_secret_key(secret_key)?; - let mut dig = Blake3Digest512::new(); - dig.update(data.get(range.clone()).ok_or_else(|| { + let in_sig_bytes = signature_digest(data.get(range.clone()).ok_or_else(|| { VeilidAPIError::invalid_argument("range is out of bounds", "range", range) })?); - let sig = dig.finalize(); - let in_sig_bytes: [u8; NONE_SIGNATURE_LENGTH] = sig.into(); let sig_out = data .get_mut(sig_idx..sig_idx + NONE_SIGNATURE_LENGTH) @@ -430,10 +430,7 @@ impl CryptoSystem for CryptoSystemNONE { self.check_public_key(public_key)?; self.check_signature(signature)?; - let mut dig = Blake3Digest512::new(); - dig.update(data); - let sig = dig.finalize(); - let in_sig_bytes: [u8; NONE_SIGNATURE_LENGTH] = sig.into(); + let in_sig_bytes = signature_digest(data); let mut verify_bytes = [0u8; NONE_SIGNATURE_LENGTH]; verify_bytes[0..32].copy_from_slice(&do_xor_32( &in_sig_bytes[0..32], @@ -466,13 +463,10 @@ impl CryptoSystem for CryptoSystemNONE { ) -> VeilidAPIResult { self.check_public_key(public_key)?; - let mut dig = Blake3Digest512::new(); - dig.update( + let in_sig_bytes = signature_digest( data.get(range) .ok_or_else(|| VeilidAPIError::internal("range is out of bounds"))?, ); - let sig = dig.finalize(); - let in_sig_bytes: [u8; NONE_SIGNATURE_LENGTH] = sig.into(); let mut verify_bytes = [0u8; NONE_SIGNATURE_LENGTH]; let signature = data .get(sig_idx..sig_idx + NONE_SIGNATURE_LENGTH) diff --git a/veilid-core/src/crypto/crypto_system/vld0/mod.rs b/veilid-core/src/crypto/crypto_system/vld0/mod.rs index bc4b4cf8..528e54c2 100644 --- a/veilid-core/src/crypto/crypto_system/vld0/mod.rs +++ b/veilid-core/src/crypto/crypto_system/vld0/mod.rs @@ -2,80 +2,45 @@ mod sizes; use super::*; -use super::hpke::{build_sealed, hpke_info, hpke_open_core, hpke_seal_core, parse_sealed}; - -use argon2::{ - password_hash::{ - phc::{PasswordHash, Salt}, - PasswordHasher, PasswordVerifier, - }, - Argon2, -}; -use chacha20::cipher::{KeyIvInit, StreamCipher}; -use chacha20::XChaCha20; -use chacha20poly1305 as ch; -use chacha20poly1305::aead::AeadInOut as _; -use chacha20poly1305::KeyInit as _; -use curve25519_dalek::digest::Digest as _; -use ed25519_dalek as ed; -use x25519_dalek as xd; - -pub use chacha20poly1305::aead::Buffer; +use super::hpke::{build_sealed, hpke_info, parse_sealed}; +use veilid_core_optimized as opt; const VLD0_DOMAIN_SIGN: &[u8] = b"VLD0_SIGN"; const VLD0_DOMAIN_CRYPT: &[u8] = b"VLD0_CRYPT"; -const VLD0_AEAD_OVERHEAD: usize = 16; +const VLD0_AEAD_OVERHEAD: usize = opt::XCHACHA20POLY1305_TAG_LENGTH; /// [`CryptoKind`] fourcc identifying the VLD0 cryptosystem. pub const CRYPTO_KIND_VLD0: CryptoKind = CryptoKind::new(*b"VLD0"); /// The VLD0 fourcc as a big-endian `u32`. pub const CRYPTO_KIND_VLD0_FOURCC: u32 = u32::from_be_bytes(*b"VLD0"); pub use sizes::*; -fn public_to_x25519_pk(public: &PublicKey) -> VeilidAPIResult { - let pk_ed = ed::VerifyingKey::from_bytes( - public - .ref_value() - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?, - ) - .map_err(VeilidAPIError::internal)?; - Ok(xd::PublicKey::from(*pk_ed.to_montgomery().as_bytes())) +fn public_to_x25519_pk(public: &PublicKey) -> VeilidAPIResult<[u8; VLD0_PUBLIC_KEY_LENGTH]> { + opt::ed25519_public_to_x25519(public.ref_value()).map_err(VeilidAPIError::internal) } -fn secret_to_x25519_sk(secret: &SecretKey) -> VeilidAPIResult { - // NOTE: ed::SigningKey.to_scalar() does not produce an unreduced scalar, we want the raw bytes here - // See https://github.com/dalek-cryptography/curve25519-dalek/issues/565 - let hash: [u8; VLD0_SIGNATURE_LENGTH] = ed::Sha512::default() - .chain_update(secret.ref_value().bytes()) - .finalize() - .into(); - let mut output = [0u8; VLD0_SECRET_KEY_LENGTH]; - output.copy_from_slice(&hash[..VLD0_SECRET_KEY_LENGTH]); - - Ok(xd::StaticSecret::from(output)) +fn secret_to_x25519_sk(secret: &SecretKey) -> VeilidAPIResult<[u8; VLD0_SECRET_KEY_LENGTH]> { + opt::ed25519_secret_to_x25519(secret.ref_value()).map_err(VeilidAPIError::internal) } pub(crate) fn vld0_generate_keypair() -> KeyPair { - let mut csprng = VeilidRng {}; - let signing_key = ed::SigningKey::generate(&mut csprng); - let verifying_key = signing_key.verifying_key(); - let public_key = BarePublicKey::new(&verifying_key.to_bytes()); - let secret_key = BareSecretKey::new(&signing_key.to_bytes()); - - KeyPair::new(CRYPTO_KIND_VLD0, BareKeyPair::new(public_key, secret_key)) + let (public_key, secret_key) = opt::ed25519_generate_keypair(&mut random_bytes); + KeyPair::new( + CRYPTO_KIND_VLD0, + BareKeyPair::new( + BarePublicKey::new(&public_key), + BareSecretKey::new(&secret_key), + ), + ) } pub(crate) fn vld0_generate_kem_keypair() -> KemKeyPair { - use ::hpke::{Kem as _, Serializable as _}; // RFC 9180 DeriveKeyPair(random ikm), the same path seal/open use - let (sk, pk) = ::hpke::kem::X25519HkdfSha256::gen_keypair_with_rng(&mut VeilidRng {}); - + let (sk, pk) = opt::hpke_x25519_generate_keypair(&mut random_bytes); KemKeyPair::new( CRYPTO_KIND_VLD0, BareKemKeyPair::new( - BareEncapsulationKey::new(&pk.to_bytes()), - BareDecapsulationKey::new(&sk.to_bytes()), + BareEncapsulationKey::new(&pk), + BareDecapsulationKey::new(&sk), ), ) } @@ -125,31 +90,18 @@ impl CryptoSystem for CryptoSystemVLD0 { instrument(level = "trace", target = "crypto", skip_all, fields(__VEILID_LOG_KEY = self.registry.log_key())) )] fn hash_password(&self, password: &[u8], salt: &[u8]) -> VeilidAPIResult { - if salt.len() < Salt::MIN_LENGTH || salt.len() > Salt::MAX_LENGTH { - apibail_generic!("invalid salt length"); - } - // Argon2 with default params (Argon2id v19) - let argon2 = Argon2::default(); - - // Hash password to PHC string ($argon2id$v=19$...) - let password_hash = argon2 - .hash_password_with_salt(password, salt) - .map_err(|e| VeilidAPIError::generic(e.to_string()))? - .to_string(); - Ok(password_hash) + opt::argon2_hash_password(password, salt).map_err(VeilidAPIError::generic) } #[cfg_attr( feature = "instrument", instrument(level = "trace", target = "crypto", skip_all, fields(__VEILID_LOG_KEY = self.registry.log_key())) )] fn verify_password(&self, password: &[u8], password_hash: &str) -> VeilidAPIResult { - let parsed_hash = PasswordHash::new(password_hash) - .map_err(|e| VeilidAPIError::parse_error(e.to_string(), password_hash))?; - // Argon2 with default params (Argon2id v19) - let argon2 = Argon2::default(); - - Ok(argon2.verify_password(password, &parsed_hash).is_ok()) + opt::argon2_verify_password(password, password_hash).map_err(|e| match e { + opt::Error::InvalidPasswordHash(msg) => VeilidAPIError::parse_error(msg, password_hash), + e => VeilidAPIError::generic(e), + }) } #[cfg_attr( @@ -157,17 +109,10 @@ impl CryptoSystem for CryptoSystemVLD0 { instrument(level = "trace", target = "crypto", skip_all, fields(__VEILID_LOG_KEY = self.registry.log_key())) )] fn derive_shared_secret(&self, password: &[u8], salt: &[u8]) -> VeilidAPIResult { - if salt.len() < Salt::MIN_LENGTH || salt.len() > Salt::MAX_LENGTH { - apibail_generic!("invalid salt length"); - } - // Argon2 with default params (Argon2id v19) - let argon2 = Argon2::default(); - let mut output_key_material = [0u8; VLD0_SHARED_SECRET_LENGTH]; - argon2 - .hash_password_into(password, salt, &mut output_key_material) - .map_err(|e| VeilidAPIError::generic(e.to_string()))?; + opt::argon2_derive_key(password, salt, &mut output_key_material) + .map_err(VeilidAPIError::generic)?; Ok(SharedSecret::new( CRYPTO_KIND_VLD0, BareSharedSecret::new(&output_key_material), @@ -202,20 +147,19 @@ impl CryptoSystem for CryptoSystemVLD0 { let pk_xd = public_to_x25519_pk(key)?; let sk_xd = secret_to_x25519_sk(secret)?; - let dh = sk_xd.diffie_hellman(&pk_xd); - if !dh.was_contributory() { - apibail_generic!("dh was not contributory"); - } - let dh_bytes = dh.to_bytes(); + let dh_bytes = opt::x25519_diffie_hellman(&sk_xd, &pk_xd).map_err(|e| match e { + opt::Error::NotContributory => VeilidAPIError::generic(e), + e => VeilidAPIError::internal(e), + })?; - let mut hasher = blake3::Hasher::new(); + let mut hasher = opt::Blake3Hasher::new(); hasher.update(VLD0_DOMAIN_CRYPT); hasher.update(&dh_bytes); let output = hasher.finalize(); Ok(SharedSecret::new( CRYPTO_KIND_VLD0, - BareSharedSecret::new(output.as_bytes()), + BareSharedSecret::new(&output), )) } @@ -229,13 +173,17 @@ impl CryptoSystem for CryptoSystemVLD0 { aad: &[u8], plaintext: &[u8], ) -> VeilidAPIResult> { - let (enc, ciphertext) = hpke_seal_core::<::hpke::kem::X25519HkdfSha256>( + let (enc, ciphertext) = opt::hpke_x25519_seal( recipient.ref_value(), &hpke_info(CRYPTO_KIND_VLD0), aad, plaintext, - &mut VeilidRng {}, - )?; + &mut random_bytes, + ) + .map_err(|e| match e { + opt::Error::InvalidKey => VeilidAPIError::invalid_argument("hpke_seal", "recipient", e), + e => VeilidAPIError::generic(e), + })?; Ok(build_sealed(CRYPTO_KIND_VLD0, &enc, &ciphertext)) } @@ -250,13 +198,17 @@ impl CryptoSystem for CryptoSystemVLD0 { sealed: &[u8], ) -> VeilidAPIResult> { let (enc, ciphertext) = parse_sealed(CRYPTO_KIND_VLD0, VLD0_HPKE_ENC_LENGTH, sealed)?; - hpke_open_core::<::hpke::kem::X25519HkdfSha256>( + opt::hpke_x25519_open( secret.ref_value(), enc, &hpke_info(CRYPTO_KIND_VLD0), aad, ciphertext, ) + .map_err(|e| match e { + opt::Error::InvalidKey => VeilidAPIError::invalid_argument("hpke_open", "secret", e), + e => VeilidAPIError::generic(e), + }) } #[cfg_attr( @@ -292,7 +244,7 @@ impl CryptoSystem for CryptoSystemVLD0 { })?; Ok(EncapsulationKey::new( CRYPTO_KIND_VLD0, - BareEncapsulationKey::new(pk_xd.as_bytes()), + BareEncapsulationKey::new(&pk_xd), )) } @@ -304,14 +256,6 @@ impl CryptoSystem for CryptoSystemVLD0 { &self, secret: &SecretKey, ) -> VeilidAPIResult { - // The sha512 conversion accepts any input length; enforce the ed25519 one - if secret.ref_value().len() != VLD0_SECRET_KEY_LENGTH { - apibail_invalid_argument!( - "decapsulation_key_from_signing_secret", - "secret", - secret.ref_value().len() - ); - } let sk_xd = secret_to_x25519_sk(secret).map_err(|e| { VeilidAPIError::invalid_argument( "decapsulation_key_from_signing_secret", @@ -321,7 +265,7 @@ impl CryptoSystem for CryptoSystemVLD0 { })?; Ok(DecapsulationKey::new( CRYPTO_KIND_VLD0, - BareDecapsulationKey::new(&sk_xd.to_bytes()), + BareDecapsulationKey::new(&sk_xd), )) } @@ -332,7 +276,7 @@ impl CryptoSystem for CryptoSystemVLD0 { fn generate_hash(&self, data: &[u8]) -> HashDigest { HashDigest::new( CRYPTO_KIND_VLD0, - BareHashDigest::new(blake3::hash(data).as_bytes()), + BareHashDigest::new(&opt::blake3_hash(data)), ) } @@ -341,11 +285,11 @@ impl CryptoSystem for CryptoSystemVLD0 { instrument(level = "trace", target = "crypto", skip_all, fields(__VEILID_LOG_KEY = self.registry.log_key())) )] fn generate_hash_reader(&self, reader: &mut dyn std::io::Read) -> VeilidAPIResult { - let mut hasher = blake3::Hasher::new(); + let mut hasher = opt::Blake3Hasher::new(); std::io::copy(reader, &mut hasher).map_err(|e| VeilidAPIError::generic(e.to_string()))?; Ok(PublicKey::new( CRYPTO_KIND_VLD0, - BarePublicKey::new(hasher.finalize().as_bytes()), + BarePublicKey::new(&hasher.finalize()), )) } @@ -410,7 +354,7 @@ impl CryptoSystem for CryptoSystemVLD0 { fn validate_hash(&self, data: &[u8], hash_digest: &HashDigest) -> VeilidAPIResult { self.check_hash_digest(hash_digest)?; - let bytes = *blake3::hash(data).as_bytes(); + let bytes = opt::blake3_hash(data); Ok(bytes == *hash_digest.ref_value().bytes()) } @@ -426,9 +370,9 @@ impl CryptoSystem for CryptoSystemVLD0 { ) -> VeilidAPIResult { self.check_hash_digest(hash_digest)?; - let mut hasher = blake3::Hasher::new(); + let mut hasher = opt::Blake3Hasher::new(); std::io::copy(reader, &mut hasher).map_err(|e| VeilidAPIError::generic(e.to_string()))?; - let bytes = *hasher.finalize().as_bytes(); + let bytes = hasher.finalize(); Ok(bytes == *hash_digest.ref_value().bytes()) } @@ -446,24 +390,21 @@ impl CryptoSystem for CryptoSystemVLD0 { self.check_public_key(public_key)?; self.check_secret_key(secret_key)?; - let mut kpb = BytesMut::with_capacity(VLD0_SECRET_KEY_LENGTH + VLD0_PUBLIC_KEY_LENGTH); + let sig_bytes = opt::ed25519_sign_prehashed( + public_key.ref_value(), + secret_key.ref_value(), + data, + VLD0_DOMAIN_SIGN, + ) + .map_err(|e| match e { + opt::Error::InvalidKey => VeilidAPIError::parse_error("Keypair is invalid", e), + e => VeilidAPIError::internal(e), + })?; - kpb.extend_from_slice(secret_key.ref_value().as_ref()); - kpb.extend_from_slice(public_key.ref_value().as_ref()); - let keypair = - ed::SigningKey::from_keypair_bytes(kpb.as_ref().try_into().unwrap_or_log()) - .map_err(|e| VeilidAPIError::parse_error("Keypair is invalid", e.to_string()))?; - - let mut dig: ed::Sha512 = ed::Sha512::default(); - dig.update(data); - - let sig_bytes = keypair - .sign_prehashed(dig, Some(VLD0_DOMAIN_SIGN)) - .map_err(VeilidAPIError::internal)?; - - let sig = Signature::new(CRYPTO_KIND_VLD0, BareSignature::new(&sig_bytes.to_bytes())); - - Ok(sig) + Ok(Signature::new( + CRYPTO_KIND_VLD0, + BareSignature::new(&sig_bytes), + )) } fn sign_in_place( @@ -477,29 +418,26 @@ impl CryptoSystem for CryptoSystemVLD0 { self.check_public_key(public_key)?; self.check_secret_key(secret_key)?; - let mut kpb = BytesMut::with_capacity(VLD0_SECRET_KEY_LENGTH + VLD0_PUBLIC_KEY_LENGTH); - - kpb.extend_from_slice(secret_key.ref_value().as_ref()); - kpb.extend_from_slice(public_key.ref_value().as_ref()); - let keypair = - ed::SigningKey::from_keypair_bytes(kpb.as_ref().try_into().unwrap_or_log()) - .map_err(|e| VeilidAPIError::parse_error("Keypair is invalid", e.to_string()))?; - - let mut dig: ed::Sha512 = ed::Sha512::default(); - - dig.update(data.get(range.clone()).ok_or_else(|| { + let message = data.get(range.clone()).ok_or_else(|| { VeilidAPIError::invalid_argument("range is out of bounds", "range", range) - })?); + })?; - let sig_bytes = keypair - .sign_prehashed(dig, Some(VLD0_DOMAIN_SIGN)) - .map_err(VeilidAPIError::internal)?; + let sig_bytes = opt::ed25519_sign_prehashed( + public_key.ref_value(), + secret_key.ref_value(), + message, + VLD0_DOMAIN_SIGN, + ) + .map_err(|e| match e { + opt::Error::InvalidKey => VeilidAPIError::parse_error("Keypair is invalid", e), + e => VeilidAPIError::internal(e), + })?; data.get_mut(sig_idx..sig_idx + VLD0_SIGNATURE_LENGTH) .ok_or_else(|| { VeilidAPIError::parse_error("signature index is out of bounds", sig_idx) })? - .copy_from_slice(&sig_bytes.to_bytes()); + .copy_from_slice(&sig_bytes); Ok(()) } @@ -517,32 +455,18 @@ impl CryptoSystem for CryptoSystemVLD0 { self.check_public_key(public_key)?; self.check_signature(signature)?; - let pk = ed::VerifyingKey::from_bytes( - public_key - .ref_value() - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?, + opt::ed25519_verify_prehashed( + public_key.ref_value(), + data, + VLD0_DOMAIN_SIGN, + signature.ref_value(), ) - .map_err(|_| VeilidAPIError::parse_error("Public key is invalid", public_key.clone()))?; - let sig = ed::Signature::from_bytes( - signature - .ref_value() - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?, - ); - - let mut dig: ed::Sha512 = ed::Sha512::default(); - dig.update(data); - - if pk - .verify_prehashed_strict(dig, Some(VLD0_DOMAIN_SIGN), &sig) - .is_err() - { - return Ok(false); - } - Ok(true) + .map_err(|e| match e { + opt::Error::InvalidKey => { + VeilidAPIError::parse_error("Public key is invalid", public_key.clone()) + } + e => VeilidAPIError::internal(e), + }) } fn verify_in_place( @@ -554,32 +478,20 @@ impl CryptoSystem for CryptoSystemVLD0 { ) -> VeilidAPIResult { self.check_public_key(public_key)?; - let pk = ed::VerifyingKey::from_bytes( - public_key - .ref_value() - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?, - ) - .map_err(|_| VeilidAPIError::parse_error("Public key is invalid", public_key.clone()))?; - - let mut dig: ed::Sha512 = ed::Sha512::default(); - dig.update( - data.get(range) - .ok_or_else(|| VeilidAPIError::internal("range is out of bounds"))?, - ); - + let message = data + .get(range) + .ok_or_else(|| VeilidAPIError::internal("range is out of bounds"))?; let sig = data .get(sig_idx..sig_idx + VLD0_SIGNATURE_LENGTH) .ok_or_else(|| VeilidAPIError::internal("signature index is out of bounds"))?; - let sig = ed::Signature::from_bytes(sig.try_into().unwrap_or_log()); - if pk - .verify_prehashed_strict(dig, Some(VLD0_DOMAIN_SIGN), &sig) - .is_err() - { - return Ok(false); - } - Ok(true) + + opt::ed25519_verify_prehashed(public_key.ref_value(), message, VLD0_DOMAIN_SIGN, sig) + .map_err(|e| match e { + opt::Error::InvalidKey => { + VeilidAPIError::parse_error("Public key is invalid", public_key.clone()) + } + e => VeilidAPIError::internal(e), + }) } // AEAD Encrypt/Decrypt @@ -596,26 +508,13 @@ impl CryptoSystem for CryptoSystemVLD0 { ) -> VeilidAPIResult<()> { self.check_shared_secret(shared_secret)?; - let shared_secret_bytes: [u8; VLD0_SHARED_SECRET_LENGTH] = shared_secret - .ref_value() - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?; - let nonce_bytes: [u8; VLD0_NONCE_LENGTH] = nonce - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?; - - let key = ch::Key::from(shared_secret_bytes); - let xnonce = ch::XNonce::from(nonce_bytes); - let aead = ch::XChaCha20Poly1305::new(&key); let mut buffer = BufferWrapper::new(body); - aead.decrypt_in_place( - &xnonce, + opt::xchacha20poly1305_decrypt_in_place( + shared_secret.ref_value(), + nonce.as_ref(), associated_data.unwrap_or(b""), - &mut buffer as &mut dyn ch::aead::Buffer, + &mut buffer, ) - .map_err(map_to_string) .map_err(VeilidAPIError::generic) } @@ -654,24 +553,14 @@ impl CryptoSystem for CryptoSystemVLD0 { self.check_nonce(nonce)?; self.check_shared_secret(shared_secret)?; - let shared_secret_bytes: [u8; VLD0_SHARED_SECRET_LENGTH] = shared_secret - .ref_value() - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?; - let nonce_bytes: [u8; VLD0_NONCE_LENGTH] = nonce - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?; - - let key = ch::Key::from(shared_secret_bytes); - let xnonce = ch::XNonce::from(nonce_bytes); - let aead = ch::XChaCha20Poly1305::new(&key); - let mut buffer = BufferWrapper::new(body); - aead.encrypt_in_place(&xnonce, associated_data.unwrap_or(b""), &mut buffer) - .map_err(map_to_string) - .map_err(VeilidAPIError::generic) + opt::xchacha20poly1305_encrypt_in_place( + shared_secret.ref_value(), + nonce.as_ref(), + associated_data.unwrap_or(b""), + &mut buffer, + ) + .map_err(VeilidAPIError::generic) } #[cfg_attr( @@ -709,21 +598,8 @@ impl CryptoSystem for CryptoSystemVLD0 { self.check_nonce(nonce)?; self.check_shared_secret(shared_secret)?; - let shared_secret_bytes: [u8; VLD0_SHARED_SECRET_LENGTH] = shared_secret - .ref_value() - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?; - let nonce_bytes: [u8; VLD0_NONCE_LENGTH] = nonce - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?; - let key = ch::Key::from(shared_secret_bytes); - let xnonce = ch::XNonce::from(nonce_bytes); - - let mut cipher = ::new(&key, &xnonce); - cipher.apply_keystream(body); - Ok(()) + opt::xchacha20_apply_keystream(shared_secret.ref_value(), nonce.as_ref(), body) + .map_err(VeilidAPIError::internal) } #[cfg_attr( @@ -740,21 +616,13 @@ impl CryptoSystem for CryptoSystemVLD0 { self.check_nonce(nonce)?; self.check_shared_secret(shared_secret)?; - let shared_secret_bytes: [u8; VLD0_SHARED_SECRET_LENGTH] = shared_secret - .ref_value() - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?; - let nonce_bytes: [u8; VLD0_NONCE_LENGTH] = nonce - .as_ref() - .try_into() - .map_err(VeilidAPIError::internal)?; - let key = ch::Key::from(shared_secret_bytes); - let xnonce = ch::XNonce::from(nonce_bytes); - - let mut cipher = ::new(&key, &xnonce); - cipher.apply_keystream_b2b(in_buf, out_buf); - Ok(()) + opt::xchacha20_apply_keystream_b2b( + shared_secret.ref_value(), + nonce.as_ref(), + in_buf, + out_buf, + ) + .map_err(VeilidAPIError::internal) } #[cfg_attr( diff --git a/veilid-core/src/crypto/mod.rs b/veilid-core/src/crypto/mod.rs index e3c60c3d..3a3e56f6 100644 --- a/veilid-core/src/crypto/mod.rs +++ b/veilid-core/src/crypto/mod.rs @@ -26,7 +26,6 @@ use stats::{CryptoStatsAccounting, MeteredCryptoSystem}; pub use types::*; use super::*; -use core::convert::TryInto; use hashlink::linked_hash_map::Entry; use hashlink::LruCache; diff --git a/veilid-core/src/crypto/tests_crypto/test_crypto_hpke.rs b/veilid-core/src/crypto/tests_crypto/test_crypto_hpke.rs index 48585269..6e2c0eef 100644 --- a/veilid-core/src/crypto/tests_crypto/test_crypto_hpke.rs +++ b/veilid-core/src/crypto/tests_crypto/test_crypto_hpke.rs @@ -1,8 +1,6 @@ use super::*; -use crate::crypto::crypto_system::hpke::{ - hpke_open_core, hpke_seal_core, HPKE_HEADER_LENGTH, HPKE_VERSION_1, -}; +use crate::crypto::crypto_system::hpke::{HPKE_HEADER_LENGTH, HPKE_VERSION_1}; const HPKE_TAG_LENGTH: usize = 16; const HPKE_ENC_LENGTH: usize = 32; @@ -38,37 +36,24 @@ fn a2_test_vector() -> A2TestVector { } // Replays fixed bytes so the vector's ikmE drives the ephemeral keypair -struct ReplayRng(Vec); -impl ::hpke::rand_core::TryRng for ReplayRng { - type Error = core::convert::Infallible; - - fn try_next_u32(&mut self) -> Result { - let mut b = [0u8; 4]; - self.try_fill_bytes(&mut b)?; - Ok(u32::from_le_bytes(b)) - } - fn try_next_u64(&mut self) -> Result { - let mut b = [0u8; 8]; - self.try_fill_bytes(&mut b)?; - Ok(u64::from_le_bytes(b)) - } - fn try_fill_bytes(&mut self, dest: &mut [u8]) -> Result<(), Self::Error> { - let rest = self.0.split_off(dest.len().min(self.0.len())); - dest[..self.0.len()].copy_from_slice(&self.0); - self.0 = rest; - Ok(()) +#[cfg(feature = "enable-crypto-vld0")] +fn replay(mut bytes: Vec) -> impl FnMut(&mut [u8]) { + move |dest: &mut [u8]| { + let n = dest.len().min(bytes.len()); + dest[..n].copy_from_slice(&bytes[..n]); + bytes.drain(..n); } } -impl ::hpke::rand_core::TryCryptoRng for ReplayRng {} -// KATs run against the seal/open core because the public methods pin veilid's own -// info string while the vectors carry the RFC's +// KATs run against the optimized crate's seal/open because the public methods pin +// veilid's own info string while the vectors carry the RFC's +#[cfg(feature = "enable-crypto-vld0")] pub fn test_hpke_kat_seal() { info!("test_hpke_kat_seal"); let v = a2_test_vector(); - let mut rng = ReplayRng(hex(&v.ikm_e)); - let (enc, ciphertext) = hpke_seal_core::<::hpke::kem::X25519HkdfSha256>( + let mut rng = replay(hex(&v.ikm_e)); + let (enc, ciphertext) = veilid_core_optimized::hpke_x25519_seal( &hex(&v.pk_rm), &hex(&v.info), &hex(&v.encryption.aad), @@ -80,18 +65,16 @@ pub fn test_hpke_kat_seal() { assert_eq!(ciphertext, hex(&v.encryption.ct)); } +#[cfg(feature = "enable-crypto-vld0")] pub fn test_hpke_kat_open() { info!("test_hpke_kat_open"); let v = a2_test_vector(); // skRm derives from ikmR per RFC 9180 DeriveKeyPair - { - use ::hpke::{Kem as _, Serializable as _}; - let (sk, _) = ::hpke::kem::X25519HkdfSha256::derive_keypair(&hex(&v.ikm_r)); - assert_eq!(sk.to_bytes().to_vec(), hex(&v.sk_rm)); - } + let (sk, _) = veilid_core_optimized::hpke_x25519_derive_keypair(&hex(&v.ikm_r)); + assert_eq!(sk.to_vec(), hex(&v.sk_rm)); - let pt = hpke_open_core::<::hpke::kem::X25519HkdfSha256>( + let pt = veilid_core_optimized::hpke_x25519_open( &hex(&v.sk_rm), &hex(&v.enc), &hex(&v.info), @@ -101,7 +84,7 @@ pub fn test_hpke_kat_open() { .unwrap(); assert_eq!(pt, hex(&v.encryption.pt)); - let wrong_aad = hpke_open_core::<::hpke::kem::X25519HkdfSha256>( + let wrong_aad = veilid_core_optimized::hpke_x25519_open( &hex(&v.sk_rm), &hex(&v.enc), &hex(&v.info), @@ -289,7 +272,6 @@ pub async fn test_hpke_bridge(vcrypto: &AsyncCryptoSystemGuard<'_>) { #[cfg(feature = "enable-crypto-vld0")] pub async fn test_hpke_bridge_matches_kem_derivation(crypto: &Crypto) { info!("test_hpke_bridge_matches_kem_derivation"); - use ::hpke::{Deserializable as _, Kem as _, Serializable as _}; let vcrypto = crypto.get_async(CRYPTO_KIND_VLD0).unwrap(); let (signing_key, signing_secret) = vcrypto.generate_keypair().await.into_split(); @@ -302,12 +284,9 @@ pub async fn test_hpke_bridge_matches_kem_derivation(crypto: &Crypto) { .await .unwrap(); - let sk = <::hpke::kem::X25519HkdfSha256 as ::hpke::Kem>::PrivateKey::from_bytes( - bridged_secret.ref_value(), - ) - .unwrap(); - let pk = ::hpke::kem::X25519HkdfSha256::sk_to_pk(&sk); - assert_eq!(pk.to_bytes().to_vec(), bridged_key.ref_value().to_vec()); + let pk = + veilid_core_optimized::hpke_x25519_public_from_secret(bridged_secret.ref_value()).unwrap(); + assert_eq!(pk.to_vec(), bridged_key.ref_value().to_vec()); } pub async fn test_hpke_wrong_recipient(vcrypto: &AsyncCryptoSystemGuard<'_>) { @@ -456,8 +435,11 @@ async fn test_hpke_cross_kind(crypto: &Crypto) { } pub async fn test_all() { - test_hpke_kat_seal(); - test_hpke_kat_open(); + #[cfg(feature = "enable-crypto-vld0")] + { + test_hpke_kat_seal(); + test_hpke_kat_open(); + } let api = crypto_tests_startup().await; let crypto = api.crypto().unwrap(); diff --git a/veilid-core/src/crypto/tests_crypto/test_crypto_timing.rs b/veilid-core/src/crypto/tests_crypto/test_crypto_timing.rs index f772cc32..35252e61 100644 --- a/veilid-core/src/crypto/tests_crypto/test_crypto_timing.rs +++ b/veilid-core/src/crypto/tests_crypto/test_crypto_timing.rs @@ -104,17 +104,15 @@ pub fn test_op_timings(cs: &CryptoSystemGuard<'_>) { cs.decapsulation_key_from_signing_secret(&kp.secret()) }), ); - // Control: the cipher crate alone, no veilid wrapper, same 64KB + // Control: the optimized crate's cipher alone, no cryptosystem wrapper, same 64KB { - use chacha20::cipher::{KeyIvInit, StreamCipher}; let mut buf = cs.random_bytes(65536); - let key = chacha20::Key::from([7u8; 32]); - let xnonce = chacha20::XNonce::from([9u8; 24]); + let key = [7u8; 32]; + let nonce = [9u8; 24]; report( - "chacha20 crate apply_keystream(64KB)", + "xchacha20_apply_keystream(64KB)", time_us(200, || { - let mut cipher = chacha20::XChaCha20::new(&key, &xnonce); - cipher.apply_keystream(&mut buf); + veilid_core_optimized::xchacha20_apply_keystream(&key, &nonce, &mut buf) }), ); } diff --git a/veilid-core/src/lib.rs b/veilid-core/src/lib.rs index 149e077f..16d38db9 100644 --- a/veilid-core/src/lib.rs +++ b/veilid-core/src/lib.rs @@ -11,7 +11,7 @@ //! - [Examples](https://gitlab.com/veilid/veilid/-/tree/main/veilid-core/examples/) //! - [API Documentation](https://docs.rs/veilid-core) //! -//! The public API is accessed by getting a [VeilidAPI] object via a call to [api_startup], [api_startup_json]. +//! The public API is accessed by getting a [VeilidAPI] object via a call to [api_startup] or [api_startup_json]. //! //! From there, a [RoutingContext] object can get you access to public and private routed operations. //! diff --git a/veilid-core/src/veilid_api/serialize_helpers/compression.rs b/veilid-core/src/veilid_api/serialize_helpers/compression.rs index 27e19d9b..b8211d69 100644 --- a/veilid-core/src/veilid_api/serialize_helpers/compression.rs +++ b/veilid-core/src/veilid_api/serialize_helpers/compression.rs @@ -1,5 +1,5 @@ use super::*; -use lz4_flex::block; +use veilid_core_optimized::{lz4_compress_prepend_size, lz4_decompress, lz4_uncompressed_size}; impl_veilid_log_facility!("compression"); @@ -9,7 +9,7 @@ impl_veilid_log_facility!("compression"); )] #[must_use] pub(crate) fn compress_prepend_size(input: &[u8]) -> Vec { - block::compress_prepend_size(input) + lz4_compress_prepend_size(input) } #[cfg_attr( @@ -17,7 +17,7 @@ pub(crate) fn compress_prepend_size(input: &[u8]) -> Vec { instrument(level = "trace", target = "compression", skip_all, err) )] pub(crate) fn decompress_size_prepended(input: &[u8], max_size: usize) -> VeilidAPIResult> { - let (uncompressed_size, input) = match block::uncompressed_size(input) { + let (uncompressed_size, input) = match lz4_uncompressed_size(input) { Ok(v) => v, Err(e) => { apibail_generic!("failed to get uncompressed size: {}", e); @@ -30,7 +30,7 @@ pub(crate) fn decompress_size_prepended(input: &[u8], max_size: usize) -> Veilid max_size ); } - match block::decompress(input, uncompressed_size) { + match lz4_decompress(input, uncompressed_size) { Ok(v) => Ok(v), Err(e) => { #[cfg(feature = "backtrace")] diff --git a/veilid-core/src/veilid_api/types/error.rs b/veilid-core/src/veilid_api/types/error.rs index afd8d2dd..a0b464a4 100644 --- a/veilid-core/src/veilid_api/types/error.rs +++ b/veilid-core/src/veilid_api/types/error.rs @@ -400,6 +400,11 @@ impl ToVeilidAPIErrorArgument for Vec { hex::encode(self) } } +impl ToVeilidAPIErrorArgument for veilid_core_optimized::Error { + fn to_veilid_api_error_argument(&self) -> String { + self.to_string() + } +} impl ToVeilidAPIErrorArgument for usize { fn to_veilid_api_error_argument(&self) -> String { diff --git a/veilid-remote-api/smoketest/Cargo.lock b/veilid-remote-api/smoketest/Cargo.lock index 20d20a26..5b5873e1 100644 --- a/veilid-remote-api/smoketest/Cargo.lock +++ b/veilid-remote-api/smoketest/Cargo.lock @@ -23,7 +23,6 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1973cfbc1a2daf9cf550e74e1f088c28e7f7d8c1e1418fb6c9dc5184b7e84c99" dependencies = [ - "bytes", "crypto-common 0.2.2", "inout 0.2.2", ] @@ -1788,9 +1787,9 @@ dependencies = [ [[package]] name = "lz4_flex" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e" +checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226" [[package]] name = "macro_rules_attribute" @@ -3200,25 +3199,18 @@ name = "veilid-core" version = "0.5.7" dependencies = [ "arc-swap", - "argon2", "async-std", "async-trait", "async-tungstenite", "backtrace", - "blake2", - "blake3", "bosion", "bytesize", "capnp", "capnpc", "cfg-if", - "chacha20", - "chacha20poly1305", "clap", - "curve25519-dalek", "data-encoding", "directories", - "ed25519-dalek", "enumset", "eyre", "flume", @@ -3228,7 +3220,6 @@ dependencies = [ "glob", "gloo-events", "hex", - "hpke", "indent", "jni", "js-sys", @@ -3238,7 +3229,6 @@ dependencies = [ "keyvaluedb-web", "lazy_static", "libc", - "lz4_flex", "macro_rules_attribute", "num-traits", "pastey", @@ -3257,6 +3247,7 @@ dependencies = [ "tracing-subscriber", "tsify", "tsify-macros", + "veilid-core-optimized", "veilid-hashlink", "veilid-tools", "wasm-bindgen", @@ -3264,10 +3255,27 @@ dependencies = [ "weak-table", "web-sys", "ws_stream_wasm", - "x25519-dalek", "zeroize", ] +[[package]] +name = "veilid-core-optimized" +version = "0.5.7" +dependencies = [ + "argon2", + "blake2", + "blake3", + "chacha20", + "chacha20poly1305", + "curve25519-dalek", + "ed25519-dalek", + "getrandom 0.4.3", + "hpke", + "lz4_flex", + "rand_core 0.10.1", + "x25519-dalek", +] + [[package]] name = "veilid-hashlink" version = "0.1.3" diff --git a/veilid-tools/src/lib.rs b/veilid-tools/src/lib.rs index 10593298..a521c71b 100644 --- a/veilid-tools/src/lib.rs +++ b/veilid-tools/src/lib.rs @@ -7,19 +7,18 @@ //! utility functions is encouraged rather than adding 'common' functionality to `veilid-core`, allowing it to //! remain free of boilerplate and utility classes that could be reused elsewhere. //! -//! Everything added to this crate must be extensively unit-tested (work in progress). +//! Everything added to this crate must be extensively unit-tested. //! //! ## Cargo features //! //! The default `veilid-tools` configurations are: //! -//! * `default` - Uses `tokio` as the async runtime +//! - `default` - Uses `tokio` as the async runtime //! //! If you use `--no-default-features`, you can switch to other runtimes: //! -//! * `rt-async-std` - Uses `async-std` as the async runtime -//! * `rt-wasm-bindgen` - When building for the `wasm32` architecture, use this to enable `wasm-bindgen-futures` as the async runtime -//! +//! - `rt-async-std` - Uses `async-std` as the async runtime +//! - `rt-wasm-bindgen` - When building for the `wasm32` architecture, use this to enable `wasm-bindgen-futures` as the async runtime #![warn(missing_docs)] #![deny(warnings)] diff --git a/version_bump.sh b/version_bump.sh index f3599018..11d3b7ff 100755 --- a/version_bump.sh +++ b/version_bump.sh @@ -34,7 +34,7 @@ NEW_VERSION=$(uv tool run bump-my-version show current_version 2>/dev/null) echo NEW_VERSION=$NEW_VERSION # Update crate dependencies for the crates we publish -cargo upgrade -p veilid-tools@$NEW_VERSION -p veilid-core@$NEW_VERSION -p veilid-remote-api@$NEW_VERSION +cargo upgrade -p veilid-tools@$NEW_VERSION -p veilid-core-optimized@$NEW_VERSION -p veilid-core@$NEW_VERSION -p veilid-remote-api@$NEW_VERSION # Update lockfile cargo update -w