From c668b03b1eaad2220d8d6160a0f5e83efa363c1a Mon Sep 17 00:00:00 2001 From: Florian Forster Date: Thu, 19 Feb 2026 00:04:17 -0800 Subject: [PATCH] chore(build): enhance proto caching and install binary plugins for improved performance, remove reliance on BSR (#11634) (cherry picked from commit 9dd2773e8004a64ad6e06a529244823b8958a46c) --- .devcontainer/devcontainer.json | 2 +- AGENTS.md | 9 +- CONTRIBUTING.md | 15 ++- apps/api/AGENTS.md | 2 + apps/api/project.json | 1 - console/AGENTS.md | 3 +- console/buf.gen.yaml | 6 +- console/project.json | 28 ++++- console/scripts/install-proto-plugins.sh | 126 +++++++++++++++++++++++ nx.json | 2 +- packages/zitadel-proto/buf.gen.yaml | 8 +- packages/zitadel-proto/project.json | 8 +- 12 files changed, 193 insertions(+), 17 deletions(-) create mode 100755 console/scripts/install-proto-plugins.sh diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1605f26da5..670a9243a2 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -46,7 +46,7 @@ "remoteEnv": { "PATH": "${containerEnv:PATH}:/workspaces/zitadel/.artifacts/bin/linux/amd64" }, - "postStartCommand": "echo 'export PATH=\"${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH):$PATH\"' >> ~/.bashrc", + "postStartCommand": "npm install -g nx@21.6.9 && echo 'export PATH=\"${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH):$PATH\"' >> ~/.bashrc", "customizations": { "vscode": { "extensions": [ diff --git a/AGENTS.md b/AGENTS.md index cf835be0a2..61223d1097 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,10 +85,13 @@ Run commands from the repository root. - Known exception: `@zitadel/console` has no configured `test` target. ## Verified Common Targets -- `@zitadel/api`: `prod`, `build`, `generate`, `lint`, `test`, `test-unit`, `test-integration` +- `@zitadel/api`: `prod`, `build`, `generate`, `generate-install`, `lint`, `test`, `test-unit`, `test-integration` - `@zitadel/login`: `dev`, `build`, `lint`, `test`, `test-unit`, `test-integration` -- `@zitadel/docs`: `dev`, `build`, `generate`, `check-links`, `check-types`, `test`, `lint` -- `@zitadel/console`: `dev`, `build`, `generate`, `lint` +- `@zitadel/docs`: `dev`, `build`, `generate`, `install-proto-plugins`, `check-links`, `check-types`, `test`, `lint` +- `@zitadel/console`: `dev`, `build`, `generate`, `install-proto-plugins`, `lint` + +## Proto Plugin Binaries +All proto plugins are installed to `.artifacts/bin///` and Nx-cached. `generate` targets wire up the correct install dependency and prepend `.artifacts/bin/` to `$PATH` — no manual install step is needed. ## PR Title Convention diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f67d4ae0d4..129b8ea484 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -49,8 +49,9 @@ You can build and start any project with Nx commands. | **Develop** | `pnpm nx run PROJECT:dev` | Development server | | | **Generate** | `pnpm nx run PROJECT:generate` | Generate .gitignored files | | | **Generate Go Files** | `pnpm nx run @zitadel/api:generate-go` | Regenerate checked-in files | This is needed to generate files using [Stringer](https://pkg.go.dev/golang.org/x/tools/cmd/stringer), [Enumer](https://github.com/dmarkham/enumer) or [gomock](https://github.com/uber-go/mock) | +| **Install Proto Plugins** | `pnpm nx run @zitadel/api:generate-install` | Install proto toolchain | Installs Go-based plugins (protoc-gen-go, connect-go, …) to `.artifacts/bin/`. Run automatically by `generate` targets; Nx caches the outputs. | | **Test - Unit** | `pnpm nx run PROJECT:test-unit` | Run unit tests | | -| **Test - Integration** | `pnpm nx run PROJECT:test-integration` | Run integration tests | Learn mnore about how to [debug API integration tests](#run-api-integration-tests) | +| **Test - Integration** | `pnpm nx run PROJECT:test-integration` | Run integration tests | Learn more about how to [debug API integration tests](#run-api-integration-tests) | | **Test - Integration Stop** | `pnpm nx run PROJECT:test-integration-stop` | Stop integration containers | | | **Test - Functional UI** | `pnpm nx run @zitadel/functional-ui:test` | Run functional UI tests | Learn more about how to [develop the Management Console and opening the interactive Test Suite](#pass-management-console-quality-checks) | | **Test - Functional UI Stop** | `pnpm nx run @zitadel/functional-ui:stop` | Run functional UI containers | | @@ -563,6 +564,18 @@ pnpm nx run @zitadel/proto:generate # Regenerate after proto changes pnpm nx run @zitadel/client:build # Build after changes ``` +### Proto Plugin Convention + +All binary proto plugins are installed to `.artifacts/bin///` and declared as Nx target outputs, making them eligible for Nx remote cache. + +| Scope | Target | Installs | +|---|---|---| +| `@zitadel/api` | `generate-install` | Go-based plugins: `buf`, `protoc-gen-go`, `protoc-gen-connect-go`, `protoc-gen-openapiv2`, `protoc-gen-validate`, `protoc-gen-authoption`, … | +| `@zitadel/console` | `install-proto-plugins` | `protoc-gen-grpc-web`, `protoc-gen-js`, `protoc-gen-openapiv2` (pre-built binaries, no Go required) | +| `@zitadel/docs` | `install-proto-plugins` | `protoc-gen-connect-openapi` (pre-built binary, no Go required) | + +`generate` targets depend on the appropriate install targets and prepend `.artifacts/bin/` to `$PATH` automatically. Running `pnpm nx run PROJECT:generate` is sufficient — no manual plugin installation needed. + ### Contribute to Docs Project documentation is located under [./apps/docs](./apps/docs). diff --git a/apps/api/AGENTS.md b/apps/api/AGENTS.md index b6b8d42fa0..48657498a9 100644 --- a/apps/api/AGENTS.md +++ b/apps/api/AGENTS.md @@ -12,6 +12,7 @@ The **API App** (`apps/api`) is the Nx application target for building and runni - **Run API (prod profile)**: `pnpm nx run @zitadel/api:prod` - **Build**: `pnpm nx run @zitadel/api:build` - **Generate (all)**: `pnpm nx run @zitadel/api:generate` +- **Install Proto Plugins**: `pnpm nx run @zitadel/api:generate-install` — installs all Go-based proto plugins (`buf`, `protoc-gen-go`, `protoc-gen-connect-go`, `protoc-gen-openapiv2`, `protoc-gen-validate`, `protoc-gen-authoption`, etc.) to `.artifacts/bin/$(GOOS)/$(GOARCH)/`. Output is Nx-cached; only reruns when plugin versions or local protoc sources change. - **Lint**: `pnpm nx run @zitadel/api:lint` - **Test (all)**: `pnpm nx run @zitadel/api:test` - **Test (unit)**: `pnpm nx run @zitadel/api:test-unit` @@ -20,3 +21,4 @@ The **API App** (`apps/api`) is the Nx application target for building and runni ## Generation Notes - `@zitadel/api:generate` can update generated, tracked files (stubs/assets/statik). Run it intentionally. - API changes in `proto/` often require regenerating API, package, and docs artifacts. +- Proto plugins are installed to `.artifacts/bin/` — do not commit these binaries; they are declared as Nx outputs and restored from cache. diff --git a/apps/api/project.json b/apps/api/project.json index c27998c70c..73c41961ac 100644 --- a/apps/api/project.json +++ b/apps/api/project.json @@ -260,7 +260,6 @@ "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@v2.22.0", "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install github.com/envoyproxy/protoc-gen-validate@v1.1.0", "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install connectrpc.com/connect/cmd/protoc-gen-connect-go@v1.18.1", - "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.35.1", "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install ./internal/protoc/protoc-gen-authoption", "GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install ./internal/protoc/protoc-gen-zitadel" ] diff --git a/console/AGENTS.md b/console/AGENTS.md index 537252304a..7f04864ac8 100644 --- a/console/AGENTS.md +++ b/console/AGENTS.md @@ -18,6 +18,7 @@ The **Management Console** (`console/`) is the administrative interface for ZITA - **Dev Server**: `pnpm nx run @zitadel/console:dev` - **Build**: `pnpm nx run @zitadel/console:build` - **Lint**: `pnpm nx run @zitadel/console:lint` -- **Generate**: `pnpm nx run @zitadel/console:generate` +- **Generate**: `pnpm nx run @zitadel/console:generate` — runs `buf generate` to produce TypeScript/JS proto stubs in `src/app/proto/generated/`. Automatically depends on `install-proto-plugins`. +- **Install Proto Plugins**: `pnpm nx run @zitadel/console:install-proto-plugins` — downloads `protoc-gen-grpc-web` v1.5.0, `protoc-gen-js` v3.21.4, and `protoc-gen-openapiv2` v2.22.0 pre-built binaries to `.artifacts/bin/`. No Go toolchain required. Output is Nx-cached. - **Test**: The `@zitadel/console` project currently has no `test` target configured in Nx. - **Functional UI Tests**: Use `pnpm nx run @zitadel/functional-ui:test` (see `tests/functional-ui/AGENTS.md`). diff --git a/console/buf.gen.yaml b/console/buf.gen.yaml index dd46b6511e..4b5641bbd5 100644 --- a/console/buf.gen.yaml +++ b/console/buf.gen.yaml @@ -3,12 +3,12 @@ version: v1 managed: enabled: true plugins: - - plugin: buf.build/protocolbuffers/js:v3.21.4 + - plugin: js out: src/app/proto/generated opt: import_style=commonjs,binary - - plugin: buf.build/grpc/web:v1.5.0 + - plugin: grpc-web out: src/app/proto/generated opt: import_style=typescript,mode=grpcweb - - plugin: buf.build/grpc-ecosystem/openapiv2:v2.27.4 + - plugin: openapiv2 out: src/app/proto/generated opt: allow_delete_body diff --git a/console/project.json b/console/project.json index 55ddd43bfa..e8565d83b1 100644 --- a/console/project.json +++ b/console/project.json @@ -17,14 +17,40 @@ }, "generate": { "description": "Generates code from protobuf definitions for API v1 endpoints", + "executor": "nx:run-commands", + "options": { + "command": "bash -c 'PATH=\"${PWD}/../.artifacts/bin/$(go env GOOS)/$(go env GOARCH):$PATH\" pnpm exec buf generate ../proto --include-imports --include-wkt'", + "cwd": "console" + }, + "dependsOn": ["install-proto-plugins"], "inputs": [ "{workspaceRoot}/proto/**/*", "{projectRoot}/buf.gen.yaml", "{projectRoot}/package.json", - "{workspaceRoot}/pnpm-lock.yaml" + "{workspaceRoot}/pnpm-lock.yaml", + { "dependentTasksOutputFiles": ".artifacts/bin/*/*/protoc-gen-js" }, + { "dependentTasksOutputFiles": ".artifacts/bin/*/*/protoc-gen-grpc-web" }, + { "dependentTasksOutputFiles": ".artifacts/bin/*/*/protoc-gen-openapiv2" } ], "outputs": ["{projectRoot}/src/app/proto/generated"] }, + "install-proto-plugins": { + "executor": "nx:run-commands", + "options": { + "command": "bash ./console/scripts/install-proto-plugins.sh" + }, + "cache": true, + "inputs": [ + { "runtime": "uname -s" }, + { "runtime": "uname -m" }, + "{projectRoot}/scripts/install-proto-plugins.sh" + ], + "outputs": [ + "{workspaceRoot}/.artifacts/bin/*/*/protoc-gen-grpc-web", + "{workspaceRoot}/.artifacts/bin/*/*/protoc-gen-js", + "{workspaceRoot}/.artifacts/bin/*/*/protoc-gen-openapiv2" + ] + }, "build": { "description": "Builds the Management Console for production", "inputs": [ diff --git a/console/scripts/install-proto-plugins.sh b/console/scripts/install-proto-plugins.sh new file mode 100755 index 0000000000..7bd5196322 --- /dev/null +++ b/console/scripts/install-proto-plugins.sh @@ -0,0 +1,126 @@ +#!/usr/bin/env bash +# Downloads protoc-gen-grpc-web, protoc-gen-js, and protoc-gen-openapiv2 +# for console protobuf generation. +# Must be run from the workspace root. +# Supports Linux (including WSL2) and macOS only. +# +# WHY binary downloads instead of BSR remote plugins: +# Using locally installed binaries avoids a network call to buf.build on every +# cache miss and removes the dependency on BSR availability during CI builds. +# Nx caches the outputs, so the downloads only happen when versions change. +# +# WHY uname instead of "go env GOOS/GOARCH": +# Keeps the script dependency-free (no Go toolchain required). +set -euo pipefail + +# ── VERSIONS & CHECKSUMS ──────────────────────────────────────────────────── +# To upgrade a plugin: update the VERSION and all four SHA256__ vars. + +GRPC_WEB_VERSION="1.5.0" +# checksums from upstream .sha256 sidecar files +GRPC_WEB_SHA256_linux_amd64="2e6e074497b221045a14d5a54e9fc910945bfdd1198b12b9fc23686a95671d64" +GRPC_WEB_SHA256_linux_arm64="522e958568cdeabdd20ef3c97394fc067ff8e374a728c08b7410bf5de8f57783" +GRPC_WEB_SHA256_darwin_amd64="1fa3ef92194d06c03448a5cba82759e9773e43d8b188866a1f1d4fc23bb1ecb7" +GRPC_WEB_SHA256_darwin_arm64="a12b759629b943ebac5528f58fac5039d9aa2fb7abb9e9684d1b481b35afbfc6" + +PROTOC_GEN_JS_VERSION="3.21.4" +# checksums of release tarballs (this release ships no upstream checksum file) +PROTOC_GEN_JS_SHA256_linux_amd64="c57ba4130471c642462fcf98c844a3c933f6c4708b9fddc859900fd2a2e72a45" +PROTOC_GEN_JS_SHA256_linux_arm64="86194b1c6baee994bb06d162887b9edace6b32a8ed971eac07fdf5d2470c6937" +PROTOC_GEN_JS_SHA256_darwin_amd64="9bfa23630fb2fd99c0328d247f91a454b4d4a2276dd4953af0a052430554510d" +PROTOC_GEN_JS_SHA256_darwin_arm64="308b3713bc6f2147c8622d0dbb82b2ffcb2e25860c89d763ea00c2d768589989" + +OPENAPIV2_VERSION="2.22.0" +# checksums from grpc-gateway_${OPENAPIV2_VERSION}_checksums.txt +OPENAPIV2_SHA256_linux_amd64="72a6fc6a6d130189c549a6d88cbdef407d3ed1c95ab101ffb3d223d8b3778c90" +OPENAPIV2_SHA256_linux_arm64="4921799b8d80dde5f8cb89817d3ae04dee1e2560e141fd0fc79a2e544cc63178" +OPENAPIV2_SHA256_darwin_amd64="14c95d1305a81822cd17ef750cfe71e8471728eba19068e9142a70a6cbaf5847" +OPENAPIV2_SHA256_darwin_arm64="dc215925f49912d53a443107879d91898b56699e5b8bc4ed0d7b9ba94939dd86" + +# ── HELPERS ────────────────────────────────────────────────────────────────── + +verify_sha256() { + local file="$1" expected="$2" + local actual + if command -v sha256sum &>/dev/null; then + actual=$(sha256sum "$file" | cut -d' ' -f1) + else + actual=$(shasum -a 256 "$file" | cut -d' ' -f1) + fi + if [ "$actual" != "$expected" ]; then + echo "ERROR: SHA256 mismatch for $(basename "$file")" >&2 + echo " expected: $expected" >&2 + echo " actual: $actual" >&2 + exit 1 + fi +} + +# ── PLATFORM DETECTION ─────────────────────────────────────────────────────── + +_uname_os=$(uname -s | tr '[:upper:]' '[:lower:]') +_uname_arch=$(uname -m) +case "$_uname_os" in + linux) GOOS="linux" ;; + darwin) GOOS="darwin" ;; + *) echo "Unsupported OS: $_uname_os" >&2; exit 1 ;; +esac +case "$_uname_arch" in + x86_64) GOARCH="amd64" ;; + aarch64 | arm64) GOARCH="arm64" ;; + *) echo "Unsupported arch: $_uname_arch" >&2; exit 1 ;; +esac + +BIN_DIR="${PWD}/.artifacts/bin/${GOOS}/${GOARCH}" +mkdir -p "$BIN_DIR" + +TMP=$(mktemp -d "${TMPDIR:-/tmp}/zitadel-console-proto-plugins.XXXXXX") +trap 'rm -rf "$TMP"' EXIT + +# ── INSTALL ─────────────────────────────────────────────────────────────────── + +# ----- protoc-gen-grpc-web (grpc/grpc-web) ----- +case "$GOARCH" in + amd64) WEB_ARCH="x86_64" ;; + arm64) WEB_ARCH="aarch64" ;; +esac +sha256_var="GRPC_WEB_SHA256_${GOOS}_${GOARCH}" +echo "Downloading protoc-gen-grpc-web v${GRPC_WEB_VERSION} (${GOOS}/${WEB_ARCH})..." +curl -fsSL \ + "https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/protoc-gen-grpc-web-${GRPC_WEB_VERSION}-${GOOS}-${WEB_ARCH}" \ + -o "${TMP}/protoc-gen-grpc-web" +verify_sha256 "${TMP}/protoc-gen-grpc-web" "${!sha256_var}" +install -m 755 "${TMP}/protoc-gen-grpc-web" "${BIN_DIR}/protoc-gen-grpc-web" + +# ----- protoc-gen-js (protocolbuffers/protobuf-javascript) ----- +# Note: macOS asset uses "osx"; ARM uses "aarch_64" (underscore, not hyphen). +case "$GOOS" in + linux) JS_OS="linux" ;; + darwin) JS_OS="osx" ;; +esac +case "$GOARCH" in + amd64) JS_ARCH="x86_64" ;; + arm64) JS_ARCH="aarch_64" ;; +esac +sha256_var="PROTOC_GEN_JS_SHA256_${GOOS}_${GOARCH}" +echo "Downloading protoc-gen-js v${PROTOC_GEN_JS_VERSION} (${JS_OS}/${JS_ARCH})..." +curl -fsSL \ + "https://github.com/protocolbuffers/protobuf-javascript/releases/download/v${PROTOC_GEN_JS_VERSION}/protobuf-javascript-${PROTOC_GEN_JS_VERSION}-${JS_OS}-${JS_ARCH}.tar.gz" \ + -o "${TMP}/p.tar.gz" +verify_sha256 "${TMP}/p.tar.gz" "${!sha256_var}" +tar -xzf "${TMP}/p.tar.gz" -C "${TMP}" +install -m 755 "${TMP}/bin/protoc-gen-js" "${BIN_DIR}/protoc-gen-js" + +# ----- protoc-gen-openapiv2 (grpc-ecosystem/grpc-gateway) ----- +case "$GOARCH" in + amd64) OAI2_ARCH="x86_64" ;; + arm64) OAI2_ARCH="arm64" ;; +esac +sha256_var="OPENAPIV2_SHA256_${GOOS}_${GOARCH}" +echo "Downloading protoc-gen-openapiv2 v${OPENAPIV2_VERSION} (${GOOS}/${OAI2_ARCH})..." +curl -fsSL \ + "https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v${OPENAPIV2_VERSION}/protoc-gen-openapiv2-v${OPENAPIV2_VERSION}-${GOOS}-${OAI2_ARCH}" \ + -o "${TMP}/protoc-gen-openapiv2" +verify_sha256 "${TMP}/protoc-gen-openapiv2" "${!sha256_var}" +install -m 755 "${TMP}/protoc-gen-openapiv2" "${BIN_DIR}/protoc-gen-openapiv2" + +echo "Console proto plugins installed to ${BIN_DIR}" diff --git a/nx.json b/nx.json index e37150af00..4ddf2bd9c6 100644 --- a/nx.json +++ b/nx.json @@ -5,7 +5,7 @@ "libsDir": "packages" }, "tui": { - "enabled": false + "enabled": true }, "namedInputs": { "default": [ diff --git a/packages/zitadel-proto/buf.gen.yaml b/packages/zitadel-proto/buf.gen.yaml index 9841083b23..7fd3712380 100644 --- a/packages/zitadel-proto/buf.gen.yaml +++ b/packages/zitadel-proto/buf.gen.yaml @@ -2,22 +2,22 @@ version: v2 managed: enabled: true plugins: - - remote: buf.build/bufbuild/es:v2.11.0 + - local: protoc-gen-es out: es include_imports: true opt: - target=js - json_types=true - import_extension=js - - remote: buf.build/bufbuild/es:v2.11.0 + - local: protoc-gen-es out: cjs include_imports: true opt: - target=js - json_types=true - import_extension=js - - js_import_style=legacy_commonjs - - remote: buf.build/bufbuild/es:v2.11.0 + - js_import_style=legacy_commonjs + - local: protoc-gen-es out: types include_imports: true opt: diff --git a/packages/zitadel-proto/project.json b/packages/zitadel-proto/project.json index 1570812203..b002a944ff 100644 --- a/packages/zitadel-proto/project.json +++ b/packages/zitadel-proto/project.json @@ -2,8 +2,14 @@ "$schema": "../../node_modules/nx/schemas/project-schema.json", "targets": { "generate": { + "executor": "nx:run-commands", + "options": { + "command": "pnpm exec buf generate ../../proto", + "cwd": "packages/zitadel-proto" + }, "inputs": [ - "default", + "{projectRoot}/buf.gen.yaml", + "{projectRoot}/package.json", "{workspaceRoot}/pnpm-lock.yaml", "{workspaceRoot}/proto/**/*" ],