mirror of
https://github.com/zitadel/zitadel.git
synced 2026-10-25 23:50:11 -05:00
chore(build): enhance proto caching and install binary plugins for improved performance, remove reliance on BSR (#11634)
(cherry picked from commit 9dd2773e80)
This commit is contained in:
Florian Forster authored and 
Livio Spring committed
1 parent
3b5947baee
commit
c668b03b1e
12 files changed
+193
-17
No files matched your search
@@ -46,7 +46,7 @@
|
||||
"remoteEnv": {
|
||||
"PATH": "${containerEnv:PATH}:/workspaces/zitadel/.artifacts/bin/linux/amd64"
|
||||
},
|
||||
"postStartCommand": "echo 'export PATH=\"${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH):$PATH\"' >> ~/.bashrc",
|
||||
"postStartCommand": "npm install -g nx@21.6.9 && echo 'export PATH=\"${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH):$PATH\"' >> ~/.bashrc",
|
||||
"customizations": {
|
||||
"vscode": {
|
||||
"extensions": [
|
||||
|
||||
@@ -85,10 +85,13 @@ Run commands from the repository root.
|
||||
- Known exception: `@zitadel/console` has no configured `test` target.
|
||||
|
||||
## Verified Common Targets
|
||||
- `@zitadel/api`: `prod`, `build`, `generate`, `lint`, `test`, `test-unit`, `test-integration`
|
||||
- `@zitadel/api`: `prod`, `build`, `generate`, `generate-install`, `lint`, `test`, `test-unit`, `test-integration`
|
||||
- `@zitadel/login`: `dev`, `build`, `lint`, `test`, `test-unit`, `test-integration`
|
||||
- `@zitadel/docs`: `dev`, `build`, `generate`, `check-links`, `check-types`, `test`, `lint`
|
||||
- `@zitadel/console`: `dev`, `build`, `generate`, `lint`
|
||||
- `@zitadel/docs`: `dev`, `build`, `generate`, `install-proto-plugins`, `check-links`, `check-types`, `test`, `lint`
|
||||
- `@zitadel/console`: `dev`, `build`, `generate`, `install-proto-plugins`, `lint`
|
||||
|
||||
## Proto Plugin Binaries
|
||||
All proto plugins are installed to `.artifacts/bin/<GOOS>/<GOARCH>/` and Nx-cached. `generate` targets wire up the correct install dependency and prepend `.artifacts/bin/` to `$PATH` — no manual install step is needed.
|
||||
|
||||
## PR Title Convention
|
||||
|
||||
|
||||
+14
-1
@@ -49,8 +49,9 @@ You can build and start any project with Nx commands.
|
||||
| **Develop** | `pnpm nx run PROJECT:dev` | Development server | |
|
||||
| **Generate** | `pnpm nx run PROJECT:generate` | Generate .gitignored files | |
|
||||
| **Generate Go Files** | `pnpm nx run @zitadel/api:generate-go` | Regenerate checked-in files | This is needed to generate files using [Stringer](https://pkg.go.dev/golang.org/x/tools/cmd/stringer), [Enumer](https://github.com/dmarkham/enumer) or [gomock](https://github.com/uber-go/mock) |
|
||||
| **Install Proto Plugins** | `pnpm nx run @zitadel/api:generate-install` | Install proto toolchain | Installs Go-based plugins (protoc-gen-go, connect-go, …) to `.artifacts/bin/`. Run automatically by `generate` targets; Nx caches the outputs. |
|
||||
| **Test - Unit** | `pnpm nx run PROJECT:test-unit` | Run unit tests | |
|
||||
| **Test - Integration** | `pnpm nx run PROJECT:test-integration` | Run integration tests | Learn mnore about how to [debug API integration tests](#run-api-integration-tests) |
|
||||
| **Test - Integration** | `pnpm nx run PROJECT:test-integration` | Run integration tests | Learn more about how to [debug API integration tests](#run-api-integration-tests) |
|
||||
| **Test - Integration Stop** | `pnpm nx run PROJECT:test-integration-stop` | Stop integration containers | |
|
||||
| **Test - Functional UI** | `pnpm nx run @zitadel/functional-ui:test` | Run functional UI tests | Learn more about how to [develop the Management Console and opening the interactive Test Suite](#pass-management-console-quality-checks) |
|
||||
| **Test - Functional UI Stop** | `pnpm nx run @zitadel/functional-ui:stop` | Run functional UI containers | |
|
||||
@@ -563,6 +564,18 @@ pnpm nx run @zitadel/proto:generate # Regenerate after proto changes
|
||||
pnpm nx run @zitadel/client:build # Build after changes
|
||||
```
|
||||
|
||||
### Proto Plugin Convention
|
||||
|
||||
All binary proto plugins are installed to `.artifacts/bin/<GOOS>/<GOARCH>/` and declared as Nx target outputs, making them eligible for Nx remote cache.
|
||||
|
||||
| Scope | Target | Installs |
|
||||
|---|---|---|
|
||||
| `@zitadel/api` | `generate-install` | Go-based plugins: `buf`, `protoc-gen-go`, `protoc-gen-connect-go`, `protoc-gen-openapiv2`, `protoc-gen-validate`, `protoc-gen-authoption`, … |
|
||||
| `@zitadel/console` | `install-proto-plugins` | `protoc-gen-grpc-web`, `protoc-gen-js`, `protoc-gen-openapiv2` (pre-built binaries, no Go required) |
|
||||
| `@zitadel/docs` | `install-proto-plugins` | `protoc-gen-connect-openapi` (pre-built binary, no Go required) |
|
||||
|
||||
`generate` targets depend on the appropriate install targets and prepend `.artifacts/bin/` to `$PATH` automatically. Running `pnpm nx run PROJECT:generate` is sufficient — no manual plugin installation needed.
|
||||
|
||||
### Contribute to Docs
|
||||
|
||||
Project documentation is located under [./apps/docs](./apps/docs).
|
||||
|
||||
@@ -12,6 +12,7 @@ The **API App** (`apps/api`) is the Nx application target for building and runni
|
||||
- **Run API (prod profile)**: `pnpm nx run @zitadel/api:prod`
|
||||
- **Build**: `pnpm nx run @zitadel/api:build`
|
||||
- **Generate (all)**: `pnpm nx run @zitadel/api:generate`
|
||||
- **Install Proto Plugins**: `pnpm nx run @zitadel/api:generate-install` — installs all Go-based proto plugins (`buf`, `protoc-gen-go`, `protoc-gen-connect-go`, `protoc-gen-openapiv2`, `protoc-gen-validate`, `protoc-gen-authoption`, etc.) to `.artifacts/bin/$(GOOS)/$(GOARCH)/`. Output is Nx-cached; only reruns when plugin versions or local protoc sources change.
|
||||
- **Lint**: `pnpm nx run @zitadel/api:lint`
|
||||
- **Test (all)**: `pnpm nx run @zitadel/api:test`
|
||||
- **Test (unit)**: `pnpm nx run @zitadel/api:test-unit`
|
||||
@@ -20,3 +21,4 @@ The **API App** (`apps/api`) is the Nx application target for building and runni
|
||||
## Generation Notes
|
||||
- `@zitadel/api:generate` can update generated, tracked files (stubs/assets/statik). Run it intentionally.
|
||||
- API changes in `proto/` often require regenerating API, package, and docs artifacts.
|
||||
- Proto plugins are installed to `.artifacts/bin/` — do not commit these binaries; they are declared as Nx outputs and restored from cache.
|
||||
@@ -260,7 +260,6 @@
|
||||
"GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install github.com/grpc-ecosystem/grpc-gateway/v2/protoc-gen-openapiv2@v2.22.0",
|
||||
"GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install github.com/envoyproxy/protoc-gen-validate@v1.1.0",
|
||||
"GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install connectrpc.com/connect/cmd/protoc-gen-connect-go@v1.18.1",
|
||||
"GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.35.1",
|
||||
"GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install ./internal/protoc/protoc-gen-authoption",
|
||||
"GOBIN=${PWD}/.artifacts/bin/$(go env GOOS)/$(go env GOARCH) go install ./internal/protoc/protoc-gen-zitadel"
|
||||
]
|
||||
|
||||
+2
-1
@@ -18,6 +18,7 @@ The **Management Console** (`console/`) is the administrative interface for ZITA
|
||||
- **Dev Server**: `pnpm nx run @zitadel/console:dev`
|
||||
- **Build**: `pnpm nx run @zitadel/console:build`
|
||||
- **Lint**: `pnpm nx run @zitadel/console:lint`
|
||||
- **Generate**: `pnpm nx run @zitadel/console:generate`
|
||||
- **Generate**: `pnpm nx run @zitadel/console:generate` — runs `buf generate` to produce TypeScript/JS proto stubs in `src/app/proto/generated/`. Automatically depends on `install-proto-plugins`.
|
||||
- **Install Proto Plugins**: `pnpm nx run @zitadel/console:install-proto-plugins` — downloads `protoc-gen-grpc-web` v1.5.0, `protoc-gen-js` v3.21.4, and `protoc-gen-openapiv2` v2.22.0 pre-built binaries to `.artifacts/bin/`. No Go toolchain required. Output is Nx-cached.
|
||||
- **Test**: The `@zitadel/console` project currently has no `test` target configured in Nx.
|
||||
- **Functional UI Tests**: Use `pnpm nx run @zitadel/functional-ui:test` (see `tests/functional-ui/AGENTS.md`).
|
||||
@@ -3,12 +3,12 @@ version: v1
|
||||
managed:
|
||||
enabled: true
|
||||
plugins:
|
||||
- plugin: buf.build/protocolbuffers/js:v3.21.4
|
||||
- plugin: js
|
||||
out: src/app/proto/generated
|
||||
opt: import_style=commonjs,binary
|
||||
- plugin: buf.build/grpc/web:v1.5.0
|
||||
- plugin: grpc-web
|
||||
out: src/app/proto/generated
|
||||
opt: import_style=typescript,mode=grpcweb
|
||||
- plugin: buf.build/grpc-ecosystem/openapiv2:v2.27.4
|
||||
- plugin: openapiv2
|
||||
out: src/app/proto/generated
|
||||
opt: allow_delete_body
|
||||
+27
-1
@@ -17,14 +17,40 @@
|
||||
},
|
||||
"generate": {
|
||||
"description": "Generates code from protobuf definitions for API v1 endpoints",
|
||||
"executor": "nx:run-commands",
|
||||
"options": {
|
||||
"command": "bash -c 'PATH=\"${PWD}/../.artifacts/bin/$(go env GOOS)/$(go env GOARCH):$PATH\" pnpm exec buf generate ../proto --include-imports --include-wkt'",
|
||||
"cwd": "console"
|
||||
},
|
||||
"dependsOn": ["install-proto-plugins"],
|
||||
"inputs": [
|
||||
"{workspaceRoot}/proto/**/*",
|
||||
"{projectRoot}/buf.gen.yaml",
|
||||
"{projectRoot}/package.json",
|
||||
"{workspaceRoot}/pnpm-lock.yaml"
|
||||
"{workspaceRoot}/pnpm-lock.yaml",
|
||||
{ "dependentTasksOutputFiles": ".artifacts/bin/*/*/protoc-gen-js" },
|
||||
{ "dependentTasksOutputFiles": ".artifacts/bin/*/*/protoc-gen-grpc-web" },
|
||||
{ "dependentTasksOutputFiles": ".artifacts/bin/*/*/protoc-gen-openapiv2" }
|
||||
],
|
||||
"outputs": ["{projectRoot}/src/app/proto/generated"]
|
||||
},
|
||||
"install-proto-plugins": {
|
||||
"executor": "nx:run-commands",
|
||||
"options": {
|
||||
"command": "bash ./console/scripts/install-proto-plugins.sh"
|
||||
},
|
||||
"cache": true,
|
||||
"inputs": [
|
||||
{ "runtime": "uname -s" },
|
||||
{ "runtime": "uname -m" },
|
||||
"{projectRoot}/scripts/install-proto-plugins.sh"
|
||||
],
|
||||
"outputs": [
|
||||
"{workspaceRoot}/.artifacts/bin/*/*/protoc-gen-grpc-web",
|
||||
"{workspaceRoot}/.artifacts/bin/*/*/protoc-gen-js",
|
||||
"{workspaceRoot}/.artifacts/bin/*/*/protoc-gen-openapiv2"
|
||||
]
|
||||
},
|
||||
"build": {
|
||||
"description": "Builds the Management Console for production",
|
||||
"inputs": [
|
||||
|
||||
Executable
+126
@@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env bash
|
||||
# Downloads protoc-gen-grpc-web, protoc-gen-js, and protoc-gen-openapiv2
|
||||
# for console protobuf generation.
|
||||
# Must be run from the workspace root.
|
||||
# Supports Linux (including WSL2) and macOS only.
|
||||
#
|
||||
# WHY binary downloads instead of BSR remote plugins:
|
||||
# Using locally installed binaries avoids a network call to buf.build on every
|
||||
# cache miss and removes the dependency on BSR availability during CI builds.
|
||||
# Nx caches the outputs, so the downloads only happen when versions change.
|
||||
#
|
||||
# WHY uname instead of "go env GOOS/GOARCH":
|
||||
# Keeps the script dependency-free (no Go toolchain required).
|
||||
set -euo pipefail
|
||||
|
||||
# ── VERSIONS & CHECKSUMS ────────────────────────────────────────────────────
|
||||
# To upgrade a plugin: update the VERSION and all four SHA256_<os>_<arch> vars.
|
||||
|
||||
GRPC_WEB_VERSION="1.5.0"
|
||||
# checksums from upstream .sha256 sidecar files
|
||||
GRPC_WEB_SHA256_linux_amd64="2e6e074497b221045a14d5a54e9fc910945bfdd1198b12b9fc23686a95671d64"
|
||||
GRPC_WEB_SHA256_linux_arm64="522e958568cdeabdd20ef3c97394fc067ff8e374a728c08b7410bf5de8f57783"
|
||||
GRPC_WEB_SHA256_darwin_amd64="1fa3ef92194d06c03448a5cba82759e9773e43d8b188866a1f1d4fc23bb1ecb7"
|
||||
GRPC_WEB_SHA256_darwin_arm64="a12b759629b943ebac5528f58fac5039d9aa2fb7abb9e9684d1b481b35afbfc6"
|
||||
|
||||
PROTOC_GEN_JS_VERSION="3.21.4"
|
||||
# checksums of release tarballs (this release ships no upstream checksum file)
|
||||
PROTOC_GEN_JS_SHA256_linux_amd64="c57ba4130471c642462fcf98c844a3c933f6c4708b9fddc859900fd2a2e72a45"
|
||||
PROTOC_GEN_JS_SHA256_linux_arm64="86194b1c6baee994bb06d162887b9edace6b32a8ed971eac07fdf5d2470c6937"
|
||||
PROTOC_GEN_JS_SHA256_darwin_amd64="9bfa23630fb2fd99c0328d247f91a454b4d4a2276dd4953af0a052430554510d"
|
||||
PROTOC_GEN_JS_SHA256_darwin_arm64="308b3713bc6f2147c8622d0dbb82b2ffcb2e25860c89d763ea00c2d768589989"
|
||||
|
||||
OPENAPIV2_VERSION="2.22.0"
|
||||
# checksums from grpc-gateway_${OPENAPIV2_VERSION}_checksums.txt
|
||||
OPENAPIV2_SHA256_linux_amd64="72a6fc6a6d130189c549a6d88cbdef407d3ed1c95ab101ffb3d223d8b3778c90"
|
||||
OPENAPIV2_SHA256_linux_arm64="4921799b8d80dde5f8cb89817d3ae04dee1e2560e141fd0fc79a2e544cc63178"
|
||||
OPENAPIV2_SHA256_darwin_amd64="14c95d1305a81822cd17ef750cfe71e8471728eba19068e9142a70a6cbaf5847"
|
||||
OPENAPIV2_SHA256_darwin_arm64="dc215925f49912d53a443107879d91898b56699e5b8bc4ed0d7b9ba94939dd86"
|
||||
|
||||
# ── HELPERS ──────────────────────────────────────────────────────────────────
|
||||
|
||||
verify_sha256() {
|
||||
local file="$1" expected="$2"
|
||||
local actual
|
||||
if command -v sha256sum &>/dev/null; then
|
||||
actual=$(sha256sum "$file" | cut -d' ' -f1)
|
||||
else
|
||||
actual=$(shasum -a 256 "$file" | cut -d' ' -f1)
|
||||
fi
|
||||
if [ "$actual" != "$expected" ]; then
|
||||
echo "ERROR: SHA256 mismatch for $(basename "$file")" >&2
|
||||
echo " expected: $expected" >&2
|
||||
echo " actual: $actual" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# ── PLATFORM DETECTION ───────────────────────────────────────────────────────
|
||||
|
||||
_uname_os=$(uname -s | tr '[:upper:]' '[:lower:]')
|
||||
_uname_arch=$(uname -m)
|
||||
case "$_uname_os" in
|
||||
linux) GOOS="linux" ;;
|
||||
darwin) GOOS="darwin" ;;
|
||||
*) echo "Unsupported OS: $_uname_os" >&2; exit 1 ;;
|
||||
esac
|
||||
case "$_uname_arch" in
|
||||
x86_64) GOARCH="amd64" ;;
|
||||
aarch64 | arm64) GOARCH="arm64" ;;
|
||||
*) echo "Unsupported arch: $_uname_arch" >&2; exit 1 ;;
|
||||
esac
|
||||
|
||||
BIN_DIR="${PWD}/.artifacts/bin/${GOOS}/${GOARCH}"
|
||||
mkdir -p "$BIN_DIR"
|
||||
|
||||
TMP=$(mktemp -d "${TMPDIR:-/tmp}/zitadel-console-proto-plugins.XXXXXX")
|
||||
trap 'rm -rf "$TMP"' EXIT
|
||||
|
||||
# ── INSTALL ───────────────────────────────────────────────────────────────────
|
||||
|
||||
# ----- protoc-gen-grpc-web (grpc/grpc-web) -----
|
||||
case "$GOARCH" in
|
||||
amd64) WEB_ARCH="x86_64" ;;
|
||||
arm64) WEB_ARCH="aarch64" ;;
|
||||
esac
|
||||
sha256_var="GRPC_WEB_SHA256_${GOOS}_${GOARCH}"
|
||||
echo "Downloading protoc-gen-grpc-web v${GRPC_WEB_VERSION} (${GOOS}/${WEB_ARCH})..."
|
||||
curl -fsSL \
|
||||
"https://github.com/grpc/grpc-web/releases/download/${GRPC_WEB_VERSION}/protoc-gen-grpc-web-${GRPC_WEB_VERSION}-${GOOS}-${WEB_ARCH}" \
|
||||
-o "${TMP}/protoc-gen-grpc-web"
|
||||
verify_sha256 "${TMP}/protoc-gen-grpc-web" "${!sha256_var}"
|
||||
install -m 755 "${TMP}/protoc-gen-grpc-web" "${BIN_DIR}/protoc-gen-grpc-web"
|
||||
|
||||
# ----- protoc-gen-js (protocolbuffers/protobuf-javascript) -----
|
||||
# Note: macOS asset uses "osx"; ARM uses "aarch_64" (underscore, not hyphen).
|
||||
case "$GOOS" in
|
||||
linux) JS_OS="linux" ;;
|
||||
darwin) JS_OS="osx" ;;
|
||||
esac
|
||||
case "$GOARCH" in
|
||||
amd64) JS_ARCH="x86_64" ;;
|
||||
arm64) JS_ARCH="aarch_64" ;;
|
||||
esac
|
||||
sha256_var="PROTOC_GEN_JS_SHA256_${GOOS}_${GOARCH}"
|
||||
echo "Downloading protoc-gen-js v${PROTOC_GEN_JS_VERSION} (${JS_OS}/${JS_ARCH})..."
|
||||
curl -fsSL \
|
||||
"https://github.com/protocolbuffers/protobuf-javascript/releases/download/v${PROTOC_GEN_JS_VERSION}/protobuf-javascript-${PROTOC_GEN_JS_VERSION}-${JS_OS}-${JS_ARCH}.tar.gz" \
|
||||
-o "${TMP}/p.tar.gz"
|
||||
verify_sha256 "${TMP}/p.tar.gz" "${!sha256_var}"
|
||||
tar -xzf "${TMP}/p.tar.gz" -C "${TMP}"
|
||||
install -m 755 "${TMP}/bin/protoc-gen-js" "${BIN_DIR}/protoc-gen-js"
|
||||
|
||||
# ----- protoc-gen-openapiv2 (grpc-ecosystem/grpc-gateway) -----
|
||||
case "$GOARCH" in
|
||||
amd64) OAI2_ARCH="x86_64" ;;
|
||||
arm64) OAI2_ARCH="arm64" ;;
|
||||
esac
|
||||
sha256_var="OPENAPIV2_SHA256_${GOOS}_${GOARCH}"
|
||||
echo "Downloading protoc-gen-openapiv2 v${OPENAPIV2_VERSION} (${GOOS}/${OAI2_ARCH})..."
|
||||
curl -fsSL \
|
||||
"https://github.com/grpc-ecosystem/grpc-gateway/releases/download/v${OPENAPIV2_VERSION}/protoc-gen-openapiv2-v${OPENAPIV2_VERSION}-${GOOS}-${OAI2_ARCH}" \
|
||||
-o "${TMP}/protoc-gen-openapiv2"
|
||||
verify_sha256 "${TMP}/protoc-gen-openapiv2" "${!sha256_var}"
|
||||
install -m 755 "${TMP}/protoc-gen-openapiv2" "${BIN_DIR}/protoc-gen-openapiv2"
|
||||
|
||||
echo "Console proto plugins installed to ${BIN_DIR}"
|
||||
@@ -5,7 +5,7 @@
|
||||
"libsDir": "packages"
|
||||
},
|
||||
"tui": {
|
||||
"enabled": false
|
||||
"enabled": true
|
||||
},
|
||||
"namedInputs": {
|
||||
"default": [
|
||||
|
||||
@@ -2,22 +2,22 @@ version: v2
|
||||
managed:
|
||||
enabled: true
|
||||
plugins:
|
||||
- remote: buf.build/bufbuild/es:v2.11.0
|
||||
- local: protoc-gen-es
|
||||
out: es
|
||||
include_imports: true
|
||||
opt:
|
||||
- target=js
|
||||
- json_types=true
|
||||
- import_extension=js
|
||||
- remote: buf.build/bufbuild/es:v2.11.0
|
||||
- local: protoc-gen-es
|
||||
out: cjs
|
||||
include_imports: true
|
||||
opt:
|
||||
- target=js
|
||||
- json_types=true
|
||||
- import_extension=js
|
||||
- js_import_style=legacy_commonjs
|
||||
- remote: buf.build/bufbuild/es:v2.11.0
|
||||
- js_import_style=legacy_commonjs
|
||||
- local: protoc-gen-es
|
||||
out: types
|
||||
include_imports: true
|
||||
opt:
|
||||
|
||||
@@ -2,8 +2,14 @@
|
||||
"$schema": "../../node_modules/nx/schemas/project-schema.json",
|
||||
"targets": {
|
||||
"generate": {
|
||||
"executor": "nx:run-commands",
|
||||
"options": {
|
||||
"command": "pnpm exec buf generate ../../proto",
|
||||
"cwd": "packages/zitadel-proto"
|
||||
},
|
||||
"inputs": [
|
||||
"default",
|
||||
"{projectRoot}/buf.gen.yaml",
|
||||
"{projectRoot}/package.json",
|
||||
"{workspaceRoot}/pnpm-lock.yaml",
|
||||
"{workspaceRoot}/proto/**/*"
|
||||
],
|
||||
|
||||
Reference in new issue
Block a user