feat(logging): add streams (#11435)

# Which Problems Are Solved

Streams allow differentiating logs produced by different components of
Zitadel.

# How the Problems Are Solved

The `backend/v3/instrumentation/logging` package now exposes convenience
function for setting and getting a logger from the context. As well as
high-level functions to emit log records at various levels. When
constructing a new logger a "stream" needs to be specified:

- **runtime**: General runtime logs, such as startup and shutdown
messages. Default for logs that do not belong to the other categories.
- **request**: Logs for incoming API and HTTP requests.
- **event_handler**: Logs for event handling in projections.
- **queue**: Logs for the job queue processing.
- **event_pusher**: Logs for event pushing to the database. Disabled by
default, contains sensitive information.

Each line from the returned logger contains a `stream` field as well as
a `version` field with the current Zitadel version.

## Runtime config

Streams can be enabled by passing an array of stream names in the
runtime config. Because some log streams may contain sensitive data
(especially events), it is now also possible to mask values by their
key.

# Additional Changes

- Wrap `slogctx` in the `logging` package. (Except API error converter
packages, because of import cycle)
- Add some docs to `logging` package so other devs understand how to add
logging to Zitadel
- Add `logging.OnError` and `logging.WithError` helper functions with
`Panic()` and `Fatal()` methods, to preserve current calls in the `cmd`
packages.
- Add instance context extractor.
- Only output request details in the request info log. Request ID
remains propagated through context.
- Moved middleware functionality into protocol specific packages. 
- Removed setting of URI to context in metric middleware. There were ony
setters and no getters. (Unused value)
- Reuse a single statusWriter in the middleware package for middlewares
that need to know the response status.

# Additional Context

- Closes #11333
- Closes #11331 
- Partly #11330

---------

Co-authored-by: Silvan <27845747+adlerhurst@users.noreply.github.com>

cmd/build/info.go

@@ -1,9 +1,8 @@
 package build
 
 import (
+	"log/slog"
 	"time"
-
-	"github.com/zitadel/logging"
 )
 
 // These variables are set via ldflags in the Makefile
@@ -21,12 +20,12 @@ func init() {
 	var err error
 	dateTime, err = time.Parse(time.RFC3339, date)
 	if err != nil {
-		logging.WithError(err).Warn("could not parse build date, using current time instead")
+		slog.Warn("could not parse build date, using current time instead", "err", err)
 		dateTime = time.Now()
 		date = dateTime.Format(time.RFC3339)
 	}
 	if version == "" {
-		logging.Warn("no build version set, using timestamp as version")
+		slog.Warn("no build version set, using timestamp as version")
 		version = date
 	}
 }

cmd/defaults.yaml

@@ -46,14 +46,36 @@ Instrumentation:
   Log:
     # Log lines lower than this level are not emitted.
     Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Streams enable logging for specific parts of the application.
+    Streams: # ZITADEL_INSTRUMENTATION_LOG_STREAMS (comma separated list)
+      - runtime # General runtime logs, such as startup and shutdown messages.
+      - request # Logs for incoming API and HTTP requests.
+      - event_handler # Logs for event handling in projections.
+      - queue # Logs for the job queue processing.
+      #- event_pusher # Logs for event pushing to the database. Warning: contains sensitive information.
+    # Mask replaces sensitive information with Value in logs matched by Key
+    Mask:
+      # Keys are the attribute keys to be masked in logs.
+      # Keys are unqualified attribute names and apply to all attributes with the specified name,
+      # regardless of their position in the attribute hierarchy.
+      # Eg. "some_key" matches "some_key" and "parent.some_key" etc.
+      # If the matched log attribute is a nested object or array,
+      # the entire structure is replaced with the specified Value.
+      Keys: # ZITADEL_INSTRUMENTATION_LOG_MASK_KEYS (comma separated list)
+        # - "first_name"
+        # - "last_name"
+      # Value is the string that replaces the original value of masked attributes.
+      Value: "****" # ZITADEL_INSTRUMENTATION_LOG_MASK_VALUE
     # Enable printing structured logs to standard error in the specified format.
+    # When disabled, the legacy Log configuration is used.
+    # Important: when legacy Log is customized, please add a format and customize before upgrading to v5
     # The following formats are supported:
-    #   - "disabled": Disables logging
+    #   - "disabled": Disables logging (or fallback to legacy Log config)
     #   - "text": Logs are printed on StdErr as human-readable text
     #   - "json": Logs are printed on StdErr as JSON objects
-    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging. Need to configure GoogleProjectID below!
-    #   - "gcp_error_reporting": JSON formatted logs compatible with Google Cloud Platform Error Reporting. Need to configure GoogleProjectID below!
-    Format: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging.
+    #   - "gcp_error_reporting": JSON formatted logs compatible with Google Cloud Platform Error Reporting.
+    Format: "disabled" # ZITADEL_INSTRUMENTATION_LOG_STDERR
     # Adds the source file and line number where the log line was emitted.
     # Not to be confused with the source of an error.
     AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE

cmd/initialise/config.go

@@ -1,34 +1,54 @@
 package initialise
 
 import (
-	"github.com/mitchellh/mapstructure"
-	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+	"errors"
+	"fmt"
 
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/id"
 )
 
 type Config struct {
-	Database database.Config
-	Machine  *id.Config
-	Log      *logging.Config
+	Instrumentation instrumentation.Config
+	Database        database.Config
+	Machine         *id.Config
+	Log             *old_logging.Config
 }
 
-func MustNewConfig(v *viper.Viper) *Config {
+func NewConfig(cmd *cobra.Command, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config := new(Config)
 	err := v.Unmarshal(config,
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
 			database.DecodeHook(false),
+			mapstructure.StringToTimeDurationHookFunc(),
 			mapstructure.TextUnmarshallerHookFunc(),
 		)),
 	)
-	logging.OnError(err).Fatal("unable to read config")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to read config: %w", err)
+	}
+
+	config.Instrumentation.Log.SetLegacyConfig(config.Log)
+	shutdown, err := instrumentation.Start(cmd.Context(), config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
+	cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamRuntime))
 
 	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
+	if err != nil {
+		err = errors.Join(err, shutdown(cmd.Context()))
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 
-	return config
+	return config, shutdown, nil
 }

cmd/initialise/init.go

@@ -3,13 +3,13 @@ package initialise
 import (
 	"context"
 	"embed"
+	"errors"
 	"fmt"
-	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -47,11 +47,15 @@ The user provided by flags needs privileges to
 `,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel init command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel init command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
-			config := MustNewConfig(viper.GetViper())
 
 			return InitAll(cmd.Context(), config)
 		},
@@ -79,7 +83,7 @@ func InitAll(ctx context.Context, config *Config) error {
 }
 
 func initialise(ctx context.Context, config database.Config, steps ...func(context.Context, *database.DB) error) error {
-	logging.Info("initialization started")
+	logging.Info(ctx, "initialization started")
 
 	err := ReadStmts()
 	if err != nil {

cmd/initialise/verify_database.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -27,11 +28,19 @@ The user provided by flags needs privileges to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel init verify database command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			err := initialise(cmd.Context(), config.Database, VerifyDatabase(config.Database.DatabaseName()))
-			logging.OnError(err).Fatal("unable to initialize the database")
+			return initialise(cmd.Context(), config.Database, VerifyDatabase(config.Database.DatabaseName()))
 		},
 	}
 }
@@ -46,10 +55,10 @@ func VerifyDatabase(databaseName string) func(context.Context, *database.DB) err
 			return fmt.Errorf("unable to get current database: %w", err)
 		}
 		if currentDatabase == databaseName {
-			logging.WithFields("database", databaseName).Info("database is same as config.database.postgres.admin.ExistingDatabase, skipping creation")
+			logging.Info(ctx, "database is same as config.database.postgres.admin.ExistingDatabase, skipping creation", "database", databaseName)
 			return nil
 		}
-		logging.WithFields("database", databaseName).Info("verify database")
+		logging.Info(ctx, "verify database", "database", databaseName)
 
 		return exec(ctx, db, fmt.Sprintf(databaseStmt, databaseName), []string{dbAlreadyExistsCode})
 	}

cmd/initialise/verify_grant.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -22,11 +23,19 @@ func newGrant() *cobra.Command {
 Prerequisites:
 - postgreSQL
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel verify grant command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			err := initialise(cmd.Context(), config.Database, VerifyGrant(config.Database.DatabaseName(), config.Database.Username()))
-			logging.OnError(err).Fatal("unable to set grant")
+			return initialise(cmd.Context(), config.Database, VerifyGrant(config.Database.DatabaseName(), config.Database.Username()))
 		},
 	}
 }
@@ -41,10 +50,10 @@ func VerifyGrant(databaseName, username string) func(context.Context, *database.
 			return fmt.Errorf("unable to get current user: %w", err)
 		}
 		if currentUser == username {
-			logging.WithFields("username", username).Info("config.database.postgres.user.username is same as config.database.postgres.admin.username, skipping grant")
+			logging.Info(ctx, "config.database.postgres.user.username is same as config.database.postgres.admin.username, skipping grant", "username", username)
 			return nil
 		}
-		logging.WithFields("user", username, "database", databaseName).Info("verify grant")
+		logging.Info(ctx, "verify grant", "user", username, "database", databaseName)
 
 		return exec(ctx, db, fmt.Sprintf(grantStmt, databaseName, username), nil)
 	}

cmd/initialise/verify_user.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -27,11 +28,19 @@ The user provided by flags needs privileges to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel verify user command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			err := initialise(cmd.Context(), config.Database, VerifyUser(config.Database.Username(), config.Database.Password()))
-			logging.OnError(err).Fatal("unable to init user")
+			return initialise(cmd.Context(), config.Database, VerifyUser(config.Database.Username(), config.Database.Password()))
 		},
 	}
 }
@@ -46,11 +55,10 @@ func VerifyUser(username, password string) func(context.Context, *database.DB) e
 			return fmt.Errorf("unable to get current user: %w", err)
 		}
 		if currentUser == username {
-			logging.WithFields("username", username).Info("config.database.postgres.user.username is same as config.database.postgres.admin.username, skipping create user")
+			logging.Info(ctx, "config.database.postgres.user.username is same as config.database.postgres.admin.username, skipping create user", "username", username)
 			return nil
 		}
-		logging.WithFields("username", username).Info("verify user")
-
+		logging.Info(ctx, "verify user", "username", username)
 		if password != "" {
 			createUserStmt += " WITH PASSWORD '" + password + "'"
 		}

cmd/initialise/verify_zitadel.go

@@ -4,12 +4,13 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	es_v3 "github.com/zitadel/zitadel/internal/eventstore/v3"
 )
@@ -23,10 +24,18 @@ func newZitadel() *cobra.Command {
 Prerequisites:
 - postgreSQL with user and database
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
-			err := verifyZitadel(cmd.Context(), config.Database)
-			logging.OnError(err).Fatal("unable to init zitadel")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel verify zitadel command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
+			return verifyZitadel(cmd.Context(), config.Database)
 		},
 	}
 }
@@ -43,32 +52,32 @@ func VerifyZitadel(ctx context.Context, db *database.DB, config database.Config)
 	}
 	defer conn.Close()
 
-	logging.WithFields().Info("verify system")
+	logging.Info(ctx, "verify system")
 	if err := exec(ctx, conn, fmt.Sprintf(createSystemStmt, config.Username()), nil); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify encryption keys")
+	logging.Info(ctx, "verify encryption keys")
 	if err := createEncryptionKeys(ctx, conn); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify projections")
+	logging.Info(ctx, "verify projections")
 	if err := exec(ctx, conn, fmt.Sprintf(createProjectionsStmt, config.Username()), nil); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify eventstore")
+	logging.Info(ctx, "verify eventstore")
 	if err := exec(ctx, conn, fmt.Sprintf(createEventstoreStmt, config.Username()), nil); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify events tables")
+	logging.Info(ctx, "verify events tables")
 	if err := createEvents(ctx, conn); err != nil {
 		return err
 	}
 
-	logging.WithFields().Info("verify unique constraints")
+	logging.Info(ctx, "verify unique constraints")
 	if err := exec(ctx, conn, createUniqueConstraints, nil); err != nil {
 		return err
 	}
@@ -77,7 +86,7 @@ func VerifyZitadel(ctx context.Context, db *database.DB, config database.Config)
 }
 
 func verifyZitadel(ctx context.Context, config database.Config) error {
-	logging.WithFields("database", config.DatabaseName()).Info("verify zitadel")
+	logging.Info(ctx, "verify zitadel", "database", config.DatabaseName())
 
 	db, err := database.Connect(config, false)
 	if err != nil {
@@ -98,7 +107,7 @@ func createEncryptionKeys(ctx context.Context, db database.Beginner) error {
 	}
 	if _, err = tx.Exec(createEncryptionKeysStmt); err != nil {
 		rollbackErr := tx.Rollback()
-		logging.OnError(rollbackErr).Error("rollback failed")
+		logging.WithError(ctx, rollbackErr).Error("rollback failed")
 		return err
 	}
 
@@ -113,7 +122,7 @@ func createEvents(ctx context.Context, conn *sql.Conn) (err error) {
 	defer func() {
 		if err != nil {
 			rollbackErr := tx.Rollback()
-			logging.OnError(rollbackErr).Error("rollback failed")
+			logging.WithError(ctx, rollbackErr).Error("rollback failed")
 			return
 		}
 		err = tx.Commit()

cmd/mirror/auth.go

@@ -11,8 +11,8 @@ import (
 	"github.com/jackc/pgx/v5/stdlib"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -24,7 +24,10 @@ func authCmd() *cobra.Command {
 ZITADEL needs to be initialized and set up with the --for-mirror flag
 Only auth requests are mirrored`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel mirror auth command failed")
+			}()
+			config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -43,11 +46,11 @@ Only auth requests are mirrored`,
 
 func copyAuth(ctx context.Context, config *Migration) {
 	sourceClient, err := database.Connect(config.Source, false)
-	logging.OnError(err).Fatal("unable to connect to source database")
+	logging.OnError(ctx, err).Fatal("unable to connect to source database")
 	defer sourceClient.Close()
 
 	destClient, err := database.Connect(config.Destination, false)
-	logging.OnError(err).Fatal("unable to connect to destination database")
+	logging.OnError(ctx, err).Fatal("unable to connect to destination database")
 	defer destClient.Close()
 
 	copyAuthRequests(ctx, sourceClient, destClient, config.MaxAuthRequestAge)
@@ -56,12 +59,12 @@ func copyAuth(ctx context.Context, config *Migration) {
 func copyAuthRequests(ctx context.Context, source, dest *database.DB, maxAuthRequestAge time.Duration) {
 	start := time.Now()
 
-	logging.Info("creating index on auth.auth_requests.change_date to speed up copy in source database")
+	logging.Info(ctx, "creating index on auth.auth_requests.change_date to speed up copy in source database")
 	_, err := source.ExecContext(ctx, "CREATE INDEX CONCURRENTLY IF NOT EXISTS auth_requests_change_date ON auth.auth_requests (change_date)")
-	logging.OnError(err).Fatal("unable to create index on auth.auth_requests.change_date")
+	logging.OnError(ctx, err).Fatal("unable to create index on auth.auth_requests.change_date")
 
 	sourceConn, err := source.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire connection")
 	defer sourceConn.Close()
 
 	r, w := io.Pipe()
@@ -78,7 +81,7 @@ func copyAuthRequests(ctx context.Context, source, dest *database.DB, maxAuthReq
 	}()
 
 	destConn, err := dest.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire connection")
 	defer destConn.Close()
 
 	var affected int64
@@ -97,7 +100,7 @@ func copyAuthRequests(ctx context.Context, source, dest *database.DB, maxAuthReq
 
 		return err
 	})
-	logging.OnError(err).Fatal("unable to copy auth requests to destination")
-	logging.OnError(<-errs).Fatal("unable to copy auth requests from source")
-	logging.WithFields("took", time.Since(start), "count", affected).Info("auth requests migrated")
+	logging.OnError(ctx, err).Fatal("unable to copy auth requests to destination")
+	logging.OnError(ctx, <-errs).Fatal("unable to copy auth requests from source")
+	logging.Info(ctx, "auth requests migrated", "took", time.Since(start), "count", affected)
 }

cmd/mirror/config.go

@@ -1,16 +1,17 @@
 package mirror
 
 import (
-	"context"
 	_ "embed"
 	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
 
 	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
 	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
@@ -28,7 +29,7 @@ type Migration struct {
 	EventBulkSize     uint32
 	MaxAuthRequestAge time.Duration
 
-	Log             *logging.Config
+	Log             *old_logging.Config
 	Machine         *id.Config
 	Instrumentation instrumentation.Config
 	Metrics         instrumentation.LegacyMetricConfig
@@ -39,39 +40,50 @@ var (
 	defaultConfig []byte
 )
 
-func mustNewMigrationConfig(ctx context.Context, v *viper.Viper) (*Migration, instrumentation.ShutdownFunc, error) {
+func newMigrationConfig(cmd *cobra.Command, v *viper.Viper) (*Migration, instrumentation.ShutdownFunc, error) {
 	config := new(Migration)
-	mustNewConfig(v, config)
-
-	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	err := newConfig(v, config)
 	if err != nil {
-		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+		return nil, nil, err
 	}
-
-	// Legacy logger
-	err = config.Log.SetLogger()
+	shutdown, err := startInstrumentation(cmd, config.Instrumentation, config.Log)
 	if err != nil {
-		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+		return nil, nil, err
 	}
-
 	id.Configure(config.Machine)
-
 	return config, shutdown, nil
 }
 
-func mustNewProjectionsConfig(v *viper.Viper) *ProjectionsConfig {
+func newProjectionsConfig(cmd *cobra.Command, v *viper.Viper) (*ProjectionsConfig, instrumentation.ShutdownFunc, error) {
 	config := new(ProjectionsConfig)
-	mustNewConfig(v, config)
-
-	err := config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
+	err := newConfig(v, config)
+	if err != nil {
+		return nil, nil, err
+	}
+	shutdown, err := startInstrumentation(cmd, config.Instrumentation, config.Log)
+	if err != nil {
+		return nil, nil, err
+	}
 	id.Configure(config.Machine)
-
-	return config
+	return config, shutdown, nil
 }
 
-func mustNewConfig(v *viper.Viper, config any) {
+func startInstrumentation(cmd *cobra.Command, cfg instrumentation.Config, logConfig *old_logging.Config) (instrumentation.ShutdownFunc, error) {
+	cfg.Log.SetLegacyConfig(logConfig)
+	shutdown, err := instrumentation.Start(cmd.Context(), cfg)
+	if err != nil {
+		return nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
+	// Legacy logger
+	err = logConfig.SetLogger()
+	if err != nil {
+		return nil, fmt.Errorf("unable to set logger: %w", err)
+	}
+	cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamRuntime))
+	return shutdown, nil
+}
+
+func newConfig(v *viper.Viper, config any) error {
 	err := v.Unmarshal(config,
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
 			hooks.SliceTypeStringDecode[*domain.CustomMessageText],
@@ -91,5 +103,8 @@ func mustNewConfig(v *viper.Viper, config any) {
 			mapstructure.TextUnmarshallerHookFunc(),
 		)),
 	)
-	logging.OnError(err).Fatal("unable to read default config")
+	if err != nil {
+		return fmt.Errorf("unable to decode config: %w", err)
+	}
+	return nil
 }

cmd/mirror/defaults.yaml

@@ -46,14 +46,36 @@ Instrumentation:
   Log:
     # Log lines lower than this level are not emitted.
     Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Streams enable logging for specific parts of the application.
+    Streams: # ZITADEL_INSTRUMENTATION_LOG_STREAMS (comma separated list)
+      - runtime # General runtime logs, such as startup and shutdown messages.
+      - request # Logs for incoming API and HTTP requests.
+      - event_handler # Logs for event handling in projections.
+      - queue # Logs for the job queue processing.
+      # - event_pusher # Logs for event pushing to the database. Warning: contains sensitive information.
+    # Mask replaces sensitive information with Value in logs matched by Key
+    Mask:
+      # Keys are the attribute keys to be masked in logs.
+      # Keys are unqualified attribute names and apply to all attributes with the specified name,
+      # regardless of their position in the attribute hierarchy.
+      # Eg. "some_key" matches "some_key" and "parent.some_key" etc.
+      # If the matched log attribute is a nested object or array,
+      # the entire structure is replaced with the specified Value.
+      Keys: # ZITADEL_INSTRUMENTATION_LOG_MASK_KEYS (comma separated list)
+        # - "first_name"
+        # - "last_name"
+      # Value is the string that replaces the original value of masked attributes.
+      Value: "****" # ZITADEL_INSTRUMENTATION_LOG_MASK_VALUE
     # Enable printing structured logs to standard error in the specified format.
+    # When disabled, the legacy Log configuration is used.
+    # Important: when legacy Log is customized, please add a format and customize before upgrading to v5
     # The following formats are supported:
-    #   - "disabled": Disables logging
+    #   - "disabled": Disables logging (or fallback to legacy Log config)
     #   - "text": Logs are printed on StdErr as human-readable text
     #   - "json": Logs are printed on StdErr as JSON objects
-    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging. Need to configure GoogleProjectID below!
-    #   - "gcp_error_reporting": JSON formatted logs compatible with Google Cloud Platform Error Reporting. Need to configure GoogleProjectID below!
-    Format: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging.
+    #   - "gcp_error_reporting": JSON formatted logs compatible with Google Cloud Platform Error Reporting.
+    Format: "disabled" # ZITADEL_INSTRUMENTATION_LOG_STDERR
     # Adds the source file and line number where the log line was emitted.
     # Not to be confused with the source of an error.
     AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
@@ -63,6 +85,10 @@ Instrumentation:
       ReportLocation: true # ZITADEL_INSTRUMENTATION_LOG_ERRORS_REPORTLOCATION
       # Adds stack traces to logged errors.
       StackTrace: false # ZITADEL_INSTRUMENTATION_LOG_ERRORS_STACKTRACE
+    # Exporter configures where the OTEL formatted logs are exported to.
+    # You typically want to disable this unless you have a OTEL collector setup.
+    # Enabling stdOut or stdErr here will result in duplicate logs (of different format)
+    # if there's also a Format set above.
     Exporter:
       # The following exporter types are supported:
       #   - "none": Disables OTEL log exporter

cmd/mirror/event_store.go

@@ -13,8 +13,8 @@ import (
 	"github.com/shopspring/decimal"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	db "github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/database/dialect"
 	"github.com/zitadel/zitadel/internal/id"
@@ -34,7 +34,11 @@ func eventstoreCmd() *cobra.Command {
 ZITADEL needs to be initialized and set up with the --for-mirror flag
 Migrate only copies events2 and unique constraints`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel mirror eventstore command failed")
+			}()
+
+			config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -54,49 +58,48 @@ Migrate only copies events2 and unique constraints`,
 
 func copyEventstore(ctx context.Context, config *Migration) {
 	sourceClient, err := db.Connect(config.Source, false)
-	logging.OnError(err).Fatal("unable to connect to source database")
+	logging.OnError(ctx, err).Fatal("unable to connect to source database")
 	defer sourceClient.Close()
 
 	destClient, err := db.Connect(config.Destination, false)
-	logging.OnError(err).Fatal("unable to connect to destination database")
+	logging.OnError(ctx, err).Fatal("unable to connect to destination database")
 	defer destClient.Close()
 
 	copyEvents(ctx, sourceClient, destClient, config.EventBulkSize)
 	copyUniqueConstraints(ctx, sourceClient, destClient)
 }
 
-func positionQuery(db *db.DB) string {
+func positionQuery(db *db.DB) (string, error) {
 	switch db.Type() {
 	case dialect.DatabaseTypePostgres:
-		return "SELECT EXTRACT(EPOCH FROM clock_timestamp())"
+		return "SELECT EXTRACT(EPOCH FROM clock_timestamp())", nil
 	case dialect.DatabaseTypeCockroach:
-		return "SELECT cluster_logical_timestamp()"
+		return "SELECT cluster_logical_timestamp()", nil
 	default:
-		logging.WithFields("db_type", db.Type()).Fatal("database type not recognized")
-		return ""
+		return "", errors.New("database type not recognized")
 	}
 }
 
 func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
-	logging.Info("starting to copy events")
+	logging.Info(ctx, "starting to copy events")
 	start := time.Now()
 	reader, writer := io.Pipe()
 
 	migrationID, err := id.SonyFlakeGenerator().Next()
-	logging.OnError(err).Fatal("unable to generate migration id")
+	logging.OnError(ctx, err).Fatal("unable to generate migration id")
 
 	sourceConn, err := source.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire source connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire source connection")
 
 	destConn, err := dest.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire dest connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire dest connection")
 
 	destinationES := eventstore.NewEventstoreFromOne(postgres.New(dest, &postgres.Config{
 		MaxRetries: 3,
 	}))
 
 	previousMigration, err := queryLastSuccessfulMigration(ctx, destinationES, source.DatabaseName())
-	logging.OnError(err).Fatal("unable to query latest successful migration")
+	logging.OnError(ctx, err).Fatal("unable to query latest successful migration")
 
 	var maxPosition decimal.Decimal
 	err = source.QueryRowContext(ctx,
@@ -105,9 +108,8 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
 		},
 		"SELECT MAX(position) FROM eventstore.events2 "+instanceClause(),
 	)
-	logging.OnError(err).Fatal("unable to query max position from source")
-
-	logging.WithFields("from", previousMigration.Position, "to", maxPosition).Info("start event migration")
+	logging.OnError(ctx, err).Fatal("unable to query max position from source")
+	logging.Info(ctx, "start event migration", "from", previousMigration.Position, "to", maxPosition)
 
 	nextPos := make(chan bool, 1)
 	pos := make(chan decimal.Decimal, 1)
@@ -140,10 +142,10 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
 				if err != nil {
 					return zerrors.ThrowUnknownf(err, "MIGRA-KTuSq", "unable to copy events from source during iteration %d", i)
 				}
-				logging.WithFields("batch_count", i).Info("batch of events copied")
+				logging.Info(ctx, "batch of events copied", "batch_count", i)
 
 				if tag.RowsAffected() < int64(bulkSize) {
-					logging.WithFields("batch_count", i).Info("last batch of events copied")
+					logging.Info(ctx, "last batch of events copied", "batch_count", i)
 					return nil
 				}
 
@@ -162,12 +164,18 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
 		defer close(pos)
 		for range nextPos {
 			var position decimal.Decimal
-			err := dest.QueryRowContext(
+			query, err := positionQuery(dest)
+			if err != nil {
+				errs <- zerrors.ThrowUnknown(err, "MIGRA-Hy6t3", "unable to generate position query")
+				return
+			}
+
+			err = dest.QueryRowContext(
 				ctx,
 				func(row *sql.Row) error {
 					return row.Scan(&position)
 				},
-				positionQuery(dest),
+				query,
 			)
 			if err != nil {
 				errs <- zerrors.ThrowUnknown(err, "MIGRA-kMyPH", "unable to query next position")
@@ -187,7 +195,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
 			pgErr := new(pgconn.PgError)
 			errors.As(err, &pgErr)
 
-			logging.WithError(err).WithField("pg_err_details", pgErr.Detail).Error("unable to copy events into destination")
+			logging.WithError(ctx, err).Error("unable to copy events into destination", "pg_err_details", pgErr.Detail)
 			return zerrors.ThrowUnknown(err, "MIGRA-DTHi7", "unable to copy events into destination")
 		}
 
@@ -197,7 +205,7 @@ func copyEvents(ctx context.Context, source, dest *db.DB, bulkSize uint32) {
 	close(errs)
 	writeCopyEventsDone(ctx, destinationES, migrationID, source.DatabaseName(), maxPosition, errs)
 
-	logging.WithFields("took", time.Since(start), "count", eventCount).Info("events migrated")
+	logging.Info(ctx, "events migrated", "took", time.Since(start), "count", eventCount)
 }
 
 func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, source string, position decimal.Decimal, errs <-chan error) {
@@ -208,24 +216,24 @@ func writeCopyEventsDone(ctx context.Context, es *eventstore.EventStore, id, sou
 	err := errors.Join(joinedErrs...)
 
 	if err != nil {
-		logging.WithError(err).Error("unable to mirror events")
+		logging.WithError(ctx, err).Error("unable to mirror events")
 		err := writeMigrationFailed(ctx, es, id, source, err)
-		logging.OnError(err).Fatal("unable to write failed event")
+		logging.OnError(ctx, err).Fatal("unable to write failed event")
 		return
 	}
 
 	err = writeMigrationSucceeded(ctx, es, id, source, position)
-	logging.OnError(err).Fatal("unable to write failed event")
+	logging.OnError(ctx, err).Fatal("unable to write succeeded event")
 }
 
 func copyUniqueConstraints(ctx context.Context, source, dest *db.DB) {
-	logging.Info("starting to copy unique constraints")
+	logging.Info(ctx, "starting to copy unique constraints")
 	start := time.Now()
 	reader, writer := io.Pipe()
 	errs := make(chan error, 1)
 
 	sourceConn, err := source.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire source connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire source connection")
 
 	go func() {
 		err := sourceConn.Raw(func(driverConn interface{}) error {
@@ -243,7 +251,7 @@ func copyUniqueConstraints(ctx context.Context, source, dest *db.DB) {
 	}()
 
 	destConn, err := dest.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire dest connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire dest connection")
 
 	var eventCount int64
 	err = destConn.Raw(func(driverConn interface{}) error {
@@ -265,7 +273,7 @@ func copyUniqueConstraints(ctx context.Context, source, dest *db.DB) {
 
 		return err
 	})
-	logging.OnError(err).Fatal("unable to copy unique constraints to destination")
-	logging.OnError(<-errs).Fatal("unable to copy unique constraints from source")
-	logging.WithFields("took", time.Since(start), "count", eventCount).Info("unique constraints migrated")
+	logging.OnError(ctx, err).Fatal("unable to copy unique constraints to destination")
+	logging.OnError(ctx, <-errs).Fatal("unable to copy unique constraints from source")
+	logging.Info(ctx, "unique constraints migrated", "took", time.Since(start), "count", eventCount)
 }

cmd/mirror/mirror.go

@@ -5,13 +5,12 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
-	"log/slog"
 	"strings"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/key"
 )
 
@@ -37,24 +36,28 @@ Order of execution:
 3. mirror event store tables
 4. recompute projections
 5. verify`,
-		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			err := viper.MergeConfig(bytes.NewBuffer(defaultConfig))
-			logging.OnError(err).Fatal("unable to read default config")
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel mirror (sub)command failed")
+			}()
 
+			err = viper.MergeConfig(bytes.NewBuffer(defaultConfig))
+			if err != nil {
+				return fmt.Errorf("unable to read default config: %w", err)
+			}
 			for _, file := range *configFiles {
 				viper.SetConfigFile(file)
 				err := viper.MergeInConfig()
-				logging.WithFields("file", file).OnError(err).Warn("unable to read config file")
+				logging.OnError(cmd.Context(), err).Error("unable to read config file")
 			}
+			return nil
 		},
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel mirror command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel mirror command failed")
 			}()
 
-			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
 			if err != nil {
 				return fmt.Errorf("unable to create migration config: %w", err)
 			}
@@ -62,7 +65,10 @@ Order of execution:
 				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
 
-			projectionConfig := mustNewProjectionsConfig(viper.GetViper())
+			projectionConfig, _, err := newProjectionsConfig(cmd, viper.GetViper())
+			if err != nil {
+				return fmt.Errorf("unable to create projections config: %w", err)
+			}
 
 			masterKey, err := key.MasterKey(cmd)
 			if err != nil {
@@ -72,7 +78,6 @@ Order of execution:
 			copySystem(cmd.Context(), config)
 			copyAuth(cmd.Context(), config)
 			copyEventstore(cmd.Context(), config)
-
 			projections(cmd.Context(), projectionConfig, masterKey)
 			return nil
 		},

cmd/mirror/projections.go

@@ -3,6 +3,7 @@ package mirror
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 	"net/http"
 	"sync"
@@ -10,8 +11,10 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/tls"
@@ -51,13 +54,24 @@ func projectionsCmd() *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "projections",
 		Short: "calls the projections synchronously",
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewProjectionsConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel mirror projections command failed")
+			}()
+			config, shutdown, err := newProjectionsConfig(cmd, viper.GetViper())
+			if err != nil {
+				return fmt.Errorf("unable to create projections config: %w", err)
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Fatal("unable to read master key")
-
+			if err != nil {
+				return fmt.Errorf("unable to read master key: %w", err)
+			}
 			projections(cmd.Context(), config, masterKey)
+			return nil
 		},
 	}
 
@@ -67,18 +81,19 @@ func projectionsCmd() *cobra.Command {
 }
 
 type ProjectionsConfig struct {
-	Destination    database.Config
-	Projections    projection.Config
-	Notifications  handlers.WorkerConfig
-	EncryptionKeys *encryption.EncryptionKeyConfig
-	SystemAPIUsers map[string]*internal_authz.SystemAPIUser
-	Eventstore     *eventstore.Config
-	Caches         *connector.CachesConfig
+	Instrumentation instrumentation.Config
+	Destination     database.Config
+	Projections     projection.Config
+	Notifications   handlers.WorkerConfig
+	EncryptionKeys  *encryption.EncryptionKeyConfig
+	SystemAPIUsers  map[string]*internal_authz.SystemAPIUser
+	Eventstore      *eventstore.Config
+	Caches          *connector.CachesConfig
 
 	Admin admin_es.Config
 	Auth  auth_es.Config
 
-	Log     *logging.Config
+	Log     *old_logging.Config
 	Machine *id.Config
 
 	ExternalPort    uint16
@@ -104,21 +119,21 @@ func projections(
 	ctx context.Context,
 	config *ProjectionsConfig,
 	masterKey string,
-) {
-	logging.Info("starting to fill projections")
+) error {
+	logging.Info(ctx, "starting to fill projections")
 	start := time.Now()
 
 	client, err := database.Connect(config.Destination, false)
-	logging.OnError(err).Fatal("unable to connect to database")
+	logging.OnError(ctx, err).Fatal("unable to connect to database")
 
 	keyStorage, err := crypto_db.NewKeyStorage(client, masterKey)
-	logging.OnError(err).Fatal("cannot start key storage")
+	logging.OnError(ctx, err).Fatal("cannot start key storage")
 
 	keys, err := encryption.EnsureEncryptionKeys(ctx, config.EncryptionKeys, keyStorage)
-	logging.OnError(err).Fatal("unable to read encryption keys")
+	logging.OnError(ctx, err).Fatal("unable to read encryption keys")
 
 	staticStorage, err := config.AssetStorage.NewStorage(client.DB)
-	logging.OnError(err).Fatal("unable create static storage")
+	logging.OnError(ctx, err).Fatal("unable create static storage")
 
 	newEventstore := new_es.NewEventstore(client)
 	config.Eventstore.Querier = old_es.NewPostgres(client)
@@ -133,7 +148,7 @@ func projections(
 	sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
 
 	cacheConnectors, err := connector.StartConnectors(config.Caches, client)
-	logging.OnError(err).Fatal("unable to start caches")
+	logging.OnError(ctx, err).Fatal("unable to start caches")
 
 	queries, err := query.StartQueries(
 		ctx,
@@ -162,10 +177,10 @@ func projections(
 		config.SystemAPIUsers,
 		false,
 	)
-	logging.OnError(err).Fatal("unable to start queries")
+	logging.OnError(ctx, err).Fatal("unable to start queries")
 
 	authZRepo, err := authz.Start(queries, es, client, keys.OIDC, config.ExternalSecure)
-	logging.OnError(err).Fatal("unable to start authz repo")
+	logging.OnError(ctx, err).Fatal("unable to start authz repo")
 
 	webAuthNConfig := &webauthn.Config{
 		DisplayName:    config.WebAuthNName,
@@ -202,10 +217,10 @@ func projections(
 		nil,
 		nil,
 	)
-	logging.OnError(err).Fatal("unable to start commands")
+	logging.OnError(ctx, err).Fatal("unable to start commands")
 
 	err = projection.Create(ctx, client, es, config.Projections, keys.OIDC, keys.SAML, config.SystemAPIUsers)
-	logging.OnError(err).Fatal("unable to start projections")
+	logging.OnError(ctx, err).Fatal("unable to start projections")
 
 	i18n.MustLoadSupportedLanguagesFromDir()
 
@@ -235,13 +250,13 @@ func projections(
 	config.Auth.Spooler.Client = client
 	config.Auth.Spooler.Eventstore = es
 	authView, err := auth_view.StartView(config.Auth.Spooler.Client, keys.OIDC, queries, config.Auth.Spooler.Eventstore)
-	logging.OnError(err).Fatal("unable to start auth view")
+	logging.OnError(ctx, err).Fatal("unable to start auth view")
 	auth_handler.Register(ctx, config.Auth.Spooler, authView, queries)
 
 	config.Admin.Spooler.Client = client
 	config.Admin.Spooler.Eventstore = es
 	adminView, err := admin_view.StartView(config.Admin.Spooler.Client)
-	logging.OnError(err).Fatal("unable to start admin view")
+	logging.OnError(ctx, err).Fatal("unable to start admin view")
 
 	admin_handler.Register(ctx, config.Admin.Spooler, adminView, staticStorage)
 
@@ -252,7 +267,7 @@ func projections(
 
 	go func() {
 		for instance := range failedInstances {
-			logging.WithFields("instance", instance).Error("projection failed")
+			logging.WithError(ctx, errors.New("projection failed for instance")).Error("projection failed", "instance", instance)
 		}
 	}()
 
@@ -263,64 +278,65 @@ func projections(
 	existingInstances := queryInstanceIDs(ctx, client)
 	for i, instance := range existingInstances {
 		instances <- instance
-		logging.WithFields("id", instance, "index", fmt.Sprintf("%d/%d", i, len(existingInstances))).Info("instance queued for projection")
+		logging.Info(ctx, "instance queued for projection", "instance", instance, "index", fmt.Sprintf("%d/%d", i, len(existingInstances)))
 	}
 	close(instances)
 	wg.Wait()
 
 	close(failedInstances)
 
-	logging.WithFields("took", time.Since(start)).Info("projections executed")
+	logging.Info(ctx, "projections executed", "took", time.Since(start))
+	return nil
 }
 
 func execProjections(ctx context.Context, instances <-chan string, failedInstances chan<- string, wg *sync.WaitGroup) {
 	for instance := range instances {
-		logging.WithFields("instance", instance).Info("starting projections")
 		ctx = internal_authz.WithInstanceID(ctx, instance)
+		logging.Info(ctx, "starting projections")
 
 		err := projection.ProjectInstance(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).WithError(err).Info("trigger failed")
+			logging.WithError(ctx, err).Error("trigger failed")
 			failedInstances <- instance
 			continue
 		}
 
 		err = projection.ProjectInstanceFields(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).WithError(err).Info("trigger fields failed")
+			logging.WithError(ctx, err).Error("trigger fields failed")
 			failedInstances <- instance
 			continue
 		}
 
 		err = admin_handler.ProjectInstance(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).WithError(err).Info("trigger admin handler failed")
+			logging.WithError(ctx, err).Error("trigger admin handler failed")
 			failedInstances <- instance
 			continue
 		}
 
 		err = projection.ProjectInstanceFields(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).WithError(err).Info("trigger fields failed")
+			logging.WithError(ctx, err).Error("trigger fields failed")
 			failedInstances <- instance
 			continue
 		}
 
 		err = auth_handler.ProjectInstance(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).WithError(err).Info("trigger auth handler failed")
+			logging.WithError(ctx, err).Error("trigger auth handler failed")
 			failedInstances <- instance
 			continue
 		}
 
 		err = notification.ProjectInstance(ctx)
 		if err != nil {
-			logging.WithFields("instance", instance).WithError(err).Info("trigger notification failed")
+			logging.WithError(ctx, err).Error("trigger notification failed")
 			failedInstances <- instance
 			continue
 		}
 
-		logging.WithFields("instance", instance).Info("projections done")
+		logging.Info(ctx, "projections done")
 	}
 	wg.Done()
 }
@@ -348,7 +364,6 @@ func queryInstanceIDs(ctx context.Context, source *database.DB) []string {
 		},
 		"SELECT DISTINCT instance_id FROM eventstore.events2 WHERE instance_id <> '' AND aggregate_type = 'instance' AND event_type = 'instance.added' AND instance_id NOT IN (SELECT instance_id FROM eventstore.events2 WHERE instance_id <> '' AND aggregate_type = 'instance' AND event_type = 'instance.removed')",
 	)
-	logging.OnError(err).Fatal("unable to query instances")
-
+	logging.OnError(ctx, err).Fatal("unable to query instances")
 	return instances
 }

cmd/mirror/system.go

@@ -10,8 +10,8 @@ import (
 	"github.com/jackc/pgx/v5/stdlib"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 )
 
@@ -23,7 +23,10 @@ func systemCmd() *cobra.Command {
 ZITADEL needs to be initialized
 Only keys and assets are mirrored`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel mirror system command failed")
+			}()
+			config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -42,11 +45,11 @@ Only keys and assets are mirrored`,
 
 func copySystem(ctx context.Context, config *Migration) {
 	sourceClient, err := database.Connect(config.Source, false)
-	logging.OnError(err).Fatal("unable to connect to source database")
+	logging.OnError(ctx, err).Fatal("unable to connect to source database")
 	defer sourceClient.Close()
 
 	destClient, err := database.Connect(config.Destination, false)
-	logging.OnError(err).Fatal("unable to connect to destination database")
+	logging.OnError(ctx, err).Fatal("unable to connect to destination database")
 	defer destClient.Close()
 
 	copyAssets(ctx, sourceClient, destClient)
@@ -54,11 +57,11 @@ func copySystem(ctx context.Context, config *Migration) {
 }
 
 func copyAssets(ctx context.Context, source, dest *database.DB) {
-	logging.Info("starting to copy assets")
+	logging.Info(ctx, "starting to copy assets")
 	start := time.Now()
 
 	sourceConn, err := source.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire source connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire source connection")
 	defer sourceConn.Close()
 
 	r, w := io.Pipe()
@@ -76,7 +79,7 @@ func copyAssets(ctx context.Context, source, dest *database.DB) {
 	}()
 
 	destConn, err := dest.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire dest connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire dest connection")
 	defer destConn.Close()
 
 	var assetCount int64
@@ -95,17 +98,17 @@ func copyAssets(ctx context.Context, source, dest *database.DB) {
 
 		return err
 	})
-	logging.OnError(err).Fatal("unable to copy assets to destination")
-	logging.OnError(<-errs).Fatal("unable to copy assets from source")
-	logging.WithFields("took", time.Since(start), "count", assetCount).Info("assets migrated")
+	logging.OnError(ctx, err).Fatal("unable to copy assets to destination")
+	logging.OnError(ctx, <-errs).Fatal("unable to copy assets from source")
+	logging.Info(ctx, "assets migrated", "took", time.Since(start), "count", assetCount)
 }
 
 func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
-	logging.Info("starting to copy encryption keys")
+	logging.Info(ctx, "starting to copy encryption keys")
 	start := time.Now()
 
 	sourceConn, err := source.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire source connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire source connection")
 	defer sourceConn.Close()
 
 	r, w := io.Pipe()
@@ -123,7 +126,7 @@ func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
 	}()
 
 	destConn, err := dest.Conn(ctx)
-	logging.OnError(err).Fatal("unable to acquire dest connection")
+	logging.OnError(ctx, err).Fatal("unable to acquire dest connection")
 	defer destConn.Close()
 
 	var keyCount int64
@@ -142,7 +145,7 @@ func copyEncryptionKeys(ctx context.Context, source, dest *database.DB) {
 
 		return err
 	})
-	logging.OnError(err).Fatal("unable to copy encryption keys to destination")
-	logging.OnError(<-errs).Fatal("unable to copy encryption keys from source")
-	logging.WithFields("took", time.Since(start), "count", keyCount).Info("encryption keys migrated")
+	logging.OnError(ctx, err).Fatal("unable to copy encryption keys to destination")
+	logging.OnError(ctx, <-errs).Fatal("unable to copy encryption keys from source")
+	logging.Info(ctx, "encryption keys migrated", "took", time.Since(start), "count", keyCount)
 }

cmd/mirror/verify.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	cryptoDatabase "github.com/zitadel/zitadel/internal/crypto/database"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/query/projection"
@@ -22,7 +22,10 @@ func verifyCmd() *cobra.Command {
 		Use:   "verify",
 		Short: "counts if source and dest have the same amount of entries",
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel mirror verify command failed")
+			}()
+			config, shutdown, err := newMigrationConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -45,11 +48,11 @@ var schemas = []string{
 
 func verifyMigration(ctx context.Context, config *Migration) {
 	sourceClient, err := database.Connect(config.Source, false)
-	logging.OnError(err).Fatal("unable to connect to source database")
+	logging.OnError(ctx, err).Fatal("unable to connect to source database")
 	defer sourceClient.Close()
 
 	destClient, err := database.Connect(config.Destination, false)
-	logging.OnError(err).Fatal("unable to connect to destination database")
+	logging.OnError(ctx, err).Fatal("unable to connect to destination database")
 	defer destClient.Close()
 
 	for _, schema := range schemas {
@@ -57,12 +60,12 @@ func verifyMigration(ctx context.Context, config *Migration) {
 			sourceCount := countEntries(ctx, sourceClient, table)
 			destCount := countEntries(ctx, destClient, table)
 
-			entry := logging.WithFields("table", table, "dest", destCount, "source", sourceCount)
+			logger := logging.FromCtx(ctx).With("table", table, "dest", destCount, "source", sourceCount)
 			if sourceCount == destCount {
-				entry.Debug("equal count")
+				logger.DebugContext(ctx, "equal count")
 				continue
 			}
-			entry.WithField("diff", destCount-sourceCount).Info("unequal count")
+			logger.InfoContext(ctx, "unequal count", "diff", destCount-sourceCount)
 		}
 	}
 }
@@ -83,7 +86,7 @@ func getTables(ctx context.Context, dest *database.DB, schema string) (tables []
 		"SELECT CONCAT(schemaname, '.', tablename) FROM pg_tables WHERE schemaname = $1",
 		schema,
 	)
-	logging.WithFields("schema", schema).OnError(err).Fatal("unable to query tables")
+	logging.OnError(ctx, err).Fatal("unable to query tables")
 	return tables
 }
 
@@ -103,7 +106,7 @@ func getViews(ctx context.Context, dest *database.DB, schema string) (tables []s
 		"SELECT CONCAT(schemaname, '.', viewname) FROM pg_views WHERE schemaname = $1",
 		schema,
 	)
-	logging.WithFields("schema", schema).OnError(err).Fatal("unable to query views")
+	logging.OnError(ctx, err).Fatal("unable to query views")
 	return tables
 }
 
@@ -125,7 +128,7 @@ func countEntries(ctx context.Context, client *database.DB, table string) (count
 		},
 		fmt.Sprintf("SELECT COUNT(*) FROM %s %s", table, instanceClause),
 	)
-	logging.WithFields("table", table, "db", client.DatabaseName()).OnError(err).Error("unable to count")
+	logging.OnError(ctx, err).Fatal("unable to count")
 
 	return count
 }

cmd/ready/config.go

@@ -1,24 +1,29 @@
 package ready
 
 import (
+	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/config/hook"
 	"github.com/zitadel/zitadel/internal/config/network"
 )
 
 type Config struct {
-	Log  *logging.Config
-	Port uint16
-	TLS  network.TLS
+	Instrumentation instrumentation.Config
+	Log             *old_logging.Config
+	Port            uint16
+	TLS             network.TLS
 }
 
-func MustNewConfig(v *viper.Viper) *Config {
+func newConfig(cmd *cobra.Command, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config := new(Config)
 	err := v.Unmarshal(config,
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
@@ -30,10 +35,23 @@ func MustNewConfig(v *viper.Viper) *Config {
 			mapstructure.TextUnmarshallerHookFunc(),
 		)),
 	)
-	logging.OnError(err).Fatal("unable to read default config")
-
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to read default config: %w", err)
+	}
+	// Force-disable metrics and tracing for ready command
+	config.Instrumentation.Metric.Exporter.Type = instrumentation.ExporterTypeNone
+	config.Instrumentation.Trace.Exporter.Type = instrumentation.ExporterTypeNone
+	// Legacy logger
 	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
-	return config
+	shutdown, err := instrumentation.Start(cmd.Context(), config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
+	cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamRuntime))
+
+	return config, shutdown, nil
 }

cmd/ready/ready.go

@@ -1,15 +1,17 @@
 package ready
 
 import (
+	"context"
 	"crypto/tls"
+	"errors"
 	"net"
 	"net/http"
-	"os"
 	"strconv"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 )
 
 func New() *cobra.Command {
@@ -17,16 +19,28 @@ func New() *cobra.Command {
 		Use:   "ready",
 		Short: "Checks if zitadel is ready",
 		Long:  "Checks if zitadel is ready",
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
-			if !ready(config) {
-				os.Exit(1)
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			// Overwrite context with ready stream for logging
+			cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamReady))
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel ready command failed")
+			}()
+			config, shutdown, err := newConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
 			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
+			if ready(cmd.Context(), config) {
+				return nil
+			}
+			return errors.New("not ready")
 		},
 	}
 }
 
-func ready(config *Config) bool {
+func ready(ctx context.Context, config *Config) bool {
 	scheme := "https"
 	if !config.TLS.Enabled {
 		scheme = "http"
@@ -35,12 +49,12 @@ func ready(config *Config) bool {
 	httpClient := http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}
 	res, err := httpClient.Get(scheme + "://" + net.JoinHostPort("localhost", strconv.Itoa(int(config.Port))) + "/debug/ready")
 	if err != nil {
-		logging.WithError(err).Warn("ready check failed")
+		logging.WithError(ctx, err).Warn("get request failed")
 		return false
 	}
 	defer res.Body.Close()
 	if res.StatusCode != 200 {
-		logging.WithFields("status", res.StatusCode).Warn("ready check failed")
+		logging.Warn(ctx, "get request failed", "status", res.StatusCode)
 		return false
 	}
 	return true

cmd/setup/10.go

@@ -7,8 +7,8 @@ import (
 	"time"
 
 	"github.com/cockroachdb/cockroach-go/v2/crdb"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -36,38 +36,39 @@ func (mig *CorrectCreationDate) Execute(ctx context.Context, _ eventstore.Event)
 	defer cancel()
 
 	for i := 0; ; i++ {
-		logging.WithFields("mig", mig.String(), "iteration", i).Debug("start iteration")
+		logCtx := logging.With(ctx, "mig", mig.String(), "iteration", i)
+		logging.Info(logCtx, "start iteration")
 		var affected int64
-		err = crdb.ExecuteTx(ctx, mig.dbClient.DB, nil, func(tx *sql.Tx) error {
-			_, err := tx.ExecContext(ctx, correctCreationDate10CreateTable)
+		err = crdb.ExecuteTx(logCtx, mig.dbClient.DB, nil, func(tx *sql.Tx) error {
+			_, err := tx.ExecContext(logCtx, correctCreationDate10CreateTable)
 			if err != nil {
 				return err
 			}
-			logging.WithFields("mig", mig.String(), "iteration", i).Debug("temp table created")
+			logging.Debug(logCtx, "temp table created")
 
-			_, err = tx.ExecContext(ctx, correctCreationDate10Truncate)
+			_, err = tx.ExecContext(logCtx, correctCreationDate10Truncate)
 			if err != nil {
 				return err
 			}
-			_, err = tx.ExecContext(ctx, correctCreationDate10FillTable)
+			_, err = tx.ExecContext(logCtx, correctCreationDate10FillTable)
 			if err != nil {
 				return err
 			}
-			logging.WithFields("mig", mig.String(), "iteration", i).Debug("temp table filled")
+			logging.Debug(logCtx, "temp table filled")
 
-			res := tx.QueryRowContext(ctx, correctCreationDate10CountWrongEvents)
+			res := tx.QueryRowContext(logCtx, correctCreationDate10CountWrongEvents)
 			if err := res.Scan(&affected); err != nil || affected == 0 {
 				return err
 			}
 
-			_, err = tx.ExecContext(ctx, correctCreationDate10Update)
+			_, err = tx.ExecContext(logCtx, correctCreationDate10Update)
 			if err != nil {
 				return err
 			}
-			logging.WithFields("mig", mig.String(), "iteration", i, "count", affected).Debug("creation dates updated")
+			logging.Debug(logCtx, "creation dates updated")
 			return nil
 		})
-		logging.WithFields("mig", mig.String(), "iteration", i).Debug("end iteration")
+		logging.Debug(logCtx, "end iteration")
 		if affected == 0 || err != nil {
 			return err
 		}

cmd/setup/14.go

@@ -8,8 +8,8 @@ import (
 	"strings"
 
 	"github.com/jackc/pgx/v5/pgconn"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -45,7 +45,7 @@ func (mig *NewEventsTable) Execute(ctx context.Context, _ eventstore.Event) erro
 	}
 	for _, stmt := range statements {
 		stmt.query = strings.ReplaceAll(stmt.query, "{{.username}}", mig.dbClient.Username())
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		_, err = mig.dbClient.ExecContext(ctx, stmt.query)
 		if err != nil {
 			return err

cmd/setup/15.go

@@ -4,8 +4,7 @@ import (
 	"context"
 	"embed"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -25,7 +24,7 @@ func (mig *CurrentProjectionState) Execute(ctx context.Context, _ eventstore.Eve
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		_, err = mig.dbClient.ExecContext(ctx, stmt.query)
 		if err != nil {
 			return err

cmd/setup/16.go

@@ -4,8 +4,7 @@ import (
 	"context"
 	_ "embed"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -25,7 +24,7 @@ func (mig *UniqueConstraintToLower) Execute(ctx context.Context, _ eventstore.Ev
 		return err
 	}
 	count, err := res.RowsAffected()
-	logging.WithFields("count", count).Info("unique constraints updated")
+	logging.Info(ctx, "unique constraints updated", "count", count)
 	return err
 }
 

cmd/setup/35.go

@@ -5,8 +5,7 @@ import (
 	"embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -26,7 +25,7 @@ func (mig *AddPositionToIndexEsWm) Execute(ctx context.Context, _ eventstore.Eve
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil {
 			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
 		}

cmd/setup/36.go

@@ -10,8 +10,8 @@ import (
 
 	"github.com/jackc/pgx/v5"
 	"github.com/jackc/pgx/v5/pgconn"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
@@ -82,19 +82,20 @@ func (mig *FillV3Milestones) pushEventsByInstance(ctx context.Context, milestone
 	slices.Sort(order)
 
 	for i, instanceID := range order {
-		logging.WithFields("instance_id", instanceID, "migration", mig.String(), "progress", fmt.Sprintf("%d/%d", i+1, len(order))).Info("filter existing milestone events")
+		logCtx := logging.With(ctx, "instance", instanceID, "migration", mig.String())
+		logging.Info(logCtx, "filter existing milestone events", "progress", fmt.Sprintf("%d/%d", i+1, len(order)))
 
 		// because each Push runs in a separate TX, we need to make sure that events
 		// from a partially executed migration are pushed again.
 		model := command.NewMilestonesReachedWriteModel(instanceID)
-		if err := mig.eventstore.FilterToQueryReducer(ctx, model); err != nil {
+		if err := mig.eventstore.FilterToQueryReducer(logCtx, model); err != nil {
 			return fmt.Errorf("milestones filter: %w", err)
 		}
 		if model.InstanceCreated {
-			logging.WithFields("instance_id", instanceID, "migration", mig.String()).Info("milestone events already migrated")
+			logging.Info(logCtx, "milestone events already migrated")
 			continue // This instance was migrated, skip
 		}
-		logging.WithFields("instance_id", instanceID, "migration", mig.String()).Info("push milestone events")
+		logging.Info(logCtx, "push milestone events")
 
 		aggregate := milestone.NewInstanceAggregate(instanceID)
 

cmd/setup/39.go

@@ -5,8 +5,7 @@ import (
 	_ "embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/repository/instance"
 )
@@ -34,7 +33,7 @@ func (mig *DeleteStaleOrgFields) Execute(ctx context.Context, _ eventstore.Event
 		return err
 	}
 	for i, instance := range instances {
-		logging.WithFields("instance_id", instance, "migration", mig.String(), "progress", fmt.Sprintf("%d/%d", i+1, len(instances))).Info("execute delete query")
+		logging.Info(ctx, "execute delete query", "instance", instance, "migration", mig.String(), "progress", fmt.Sprintf("%d/%d", i+1, len(instances)))
 		if _, err := mig.eventstore.Client().ExecContext(ctx, deleteStaleOrgFields, instance); err != nil {
 			return err
 		}

cmd/setup/40.go

@@ -10,8 +10,7 @@ import (
 	"strings"
 	"text/template"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -39,7 +38,7 @@ func (mig *InitPushFunc) Execute(ctx context.Context, _ eventstore.Event) (err e
 	}
 	defer func() {
 		closeErr := conn.Close()
-		logging.OnError(closeErr).Debug("failed to release connection")
+		logging.OnError(ctx, closeErr).Debug("failed to release connection")
 		// Force the pool to reopen connections to apply the new types
 		mig.dbClient.Pool.Reset()
 	}()
@@ -48,7 +47,7 @@ func (mig *InitPushFunc) Execute(ctx context.Context, _ eventstore.Event) (err e
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		if _, err := conn.ExecContext(ctx, stmt.query); err != nil {
 			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
 		}

cmd/setup/43.go

@@ -5,8 +5,7 @@ import (
 	"embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -26,7 +25,7 @@ func (mig *CreateFieldsDomainIndex) Execute(ctx context.Context, _ eventstore.Ev
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil {
 			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
 		}

cmd/setup/44.go

@@ -5,8 +5,7 @@ import (
 	"embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -26,7 +25,7 @@ func (mig *ReplaceCurrentSequencesIndex) Execute(ctx context.Context, _ eventsto
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil {
 			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
 		}

cmd/setup/45.go

@@ -6,8 +6,7 @@ import (
 	"encoding/json"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	"github.com/zitadel/zitadel/internal/query/projection"
@@ -42,7 +41,7 @@ func (mig *CorrectProjectOwners) Execute(ctx context.Context, _ eventstore.Event
 	ctx = authz.SetCtxData(ctx, authz.CtxData{UserID: "SETUP"})
 	for i, instance := range instances {
 		ctx = authz.WithInstanceID(ctx, instance)
-		logging.WithFields("instance_id", instance, "migration", mig.String(), "progress", fmt.Sprintf("%d/%d", i+1, len(instances))).Info("correct owners of projects")
+		logging.Info(ctx, "correct owners of projects", "instance", instance, "migration", mig.String(), "progress", fmt.Sprintf("%d/%d", i+1, len(instances)))
 		didCorrect, err := mig.correctInstanceProjects(ctx, instance)
 		if err != nil {
 			return err
@@ -51,7 +50,7 @@ func (mig *CorrectProjectOwners) Execute(ctx context.Context, _ eventstore.Event
 			continue
 		}
 		_, err = projection.ProjectGrantProjection.Trigger(ctx)
-		logging.OnError(err).Debug("failed triggering project grant projection to update owners")
+		logging.OnError(ctx, err).Debug("failed triggering project grant projection to update owners")
 	}
 	return nil
 }

cmd/setup/46.go

@@ -5,8 +5,7 @@ import (
 	"embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -26,7 +25,7 @@ func (mig *InitPermissionFunctions) Execute(ctx context.Context, _ eventstore.Ev
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		if _, err := mig.eventstoreClient.ExecContext(ctx, stmt.query); err != nil {
 			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
 		}

cmd/setup/49.go

@@ -5,8 +5,7 @@ import (
 	"embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -26,7 +25,7 @@ func (mig *InitPermittedOrgsFunction) Execute(ctx context.Context, _ eventstore.
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		if _, err := mig.eventstoreClient.ExecContext(ctx, stmt.query); err != nil {
 			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
 		}

cmd/setup/53.go

@@ -5,8 +5,7 @@ import (
 	"embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -24,7 +23,7 @@ func (mig *InitPermittedOrgsFunction53) Execute(ctx context.Context, _ eventstor
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil {
 			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
 		}

cmd/setup/58.go

@@ -6,8 +6,7 @@ import (
 	"embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 )
@@ -36,7 +35,7 @@ func (mig *ReplaceLoginNames3View) Execute(ctx context.Context, _ eventstore.Eve
 		return err
 	}
 	for _, stmt := range statements {
-		logging.WithFields("file", stmt.file, "migration", mig.String()).Info("execute statement")
+		logging.Info(ctx, "execute statement", "file", stmt.file, "migration", mig.String())
 		if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil {
 			return fmt.Errorf("%s %s: %w", mig.String(), stmt.file, err)
 		}

cmd/setup/59.go

@@ -4,8 +4,7 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/crypto"
@@ -41,7 +40,7 @@ func (mig *SetupWebkeys) Execute(ctx context.Context, _ eventstore.Event) error
 
 	for _, instance := range instances {
 		ctx := authz.WithInstanceID(ctx, instance)
-		logging.Info("prepare initial webkeys for instance", "instance_id", instance, "migration", mig)
+		logging.Info(ctx, "prepare initial webkeys for instance", "instance", instance, "migration", mig)
 		if err := mig.commands.GenerateInitialWebKeys(ctx, conf); err != nil {
 			return fmt.Errorf("%s generate initial webkeys: %w", mig, err)
 		}

cmd/setup/cleanup.go

@@ -3,11 +3,12 @@ package setup
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/eventstore"
 	old_es "github.com/zitadel/zitadel/internal/eventstore/repository/sql"
@@ -21,39 +22,48 @@ func NewCleanup() *cobra.Command {
 		Short: "cleans up migration if they got stuck",
 		Long:  `cleans up migration if they got stuck`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
-			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			defer func() {
+				logging.OnError(cmd.Context(), err).Error("zitadel setup cleanup command failed")
+			}()
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
 			defer func() {
 				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
-			Cleanup(cmd.Context(), config)
-			return nil
+			return Cleanup(cmd.Context(), config)
 		},
 	}
 }
 
-func Cleanup(ctx context.Context, config *Config) {
-	logging.Info("cleanup started")
+func Cleanup(ctx context.Context, config *Config) error {
+	logging.Info(ctx, "cleanup started")
 
 	dbClient, err := database.Connect(config.Database, false)
-	logging.OnError(err).Fatal("unable to connect to database")
+	if err != nil {
+		return fmt.Errorf("unable to connect to database: %w", err)
+	}
 
 	config.Eventstore.Pusher = new_es.NewEventstore(dbClient)
 	config.Eventstore.Querier = old_es.NewPostgres(dbClient)
 	es := eventstore.NewEventstore(config.Eventstore)
 
 	step, err := migration.LastStuckStep(ctx, es)
-	logging.OnError(err).Fatal("unable to query latest migration")
-
-	if step == nil {
-		logging.Info("there is no stuck migration please run `zitadel setup`")
-		return
+	if err != nil {
+		return fmt.Errorf("unable to query latest migration: %w", err)
 	}
 
-	logging.WithFields("name", step.Name).Info("cleanup migration")
+	if step == nil {
+		logging.Info(ctx, "there is no stuck migration please run `zitadel setup`")
+		return nil
+	}
+
+	logging.Info(ctx, "cleanup migration", "name", step.Name)
 
 	err = migration.CancelStep(ctx, es, step)
-	logging.OnError(err).Fatal("cleanup migration failed please retry")
+	if err != nil {
+		return fmt.Errorf("cleanup migration failed please retry: %w", err)
+	}
+	return nil
 }

cmd/setup/config.go

@@ -9,10 +9,12 @@ import (
 	"time"
 
 	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
 
 	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -44,7 +46,7 @@ type Config struct {
 	ExternalPort    uint16
 	ExternalSecure  bool
 	Instrumentation instrumentation.Config
-	Log             *logging.Config
+	Log             *old_logging.Config
 	Metrics         *instrumentation.LegacyMetricConfig
 	EncryptionKeys  *encryption.EncryptionKeyConfig
 	DefaultInstance command.InstanceSetup
@@ -70,7 +72,7 @@ type InitProjections struct {
 	BulkLimit        uint64
 }
 
-func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
+func NewConfig(cmd *cobra.Command, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config := new(Config)
 	err := v.Unmarshal(config,
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
@@ -94,14 +96,16 @@ func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.Sh
 	}
 
 	config.Instrumentation.Metric.SetLegacyConfig(config.Metrics)
-	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	config.Instrumentation.Log.SetLegacyConfig(config.Log)
+	shutdown, err := instrumentation.Start(cmd.Context(), config.Instrumentation)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
 	}
+	cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamReady))
 
 	err = config.Log.SetLogger()
 	if err != nil {
-		err = errors.Join(err, shutdown(ctx))
+		err = errors.Join(err, shutdown(cmd.Context()))
 		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
 	}
 
@@ -180,18 +184,20 @@ type Steps struct {
 	s69CacheTablesLogged                    *CacheTablesLogged
 }
 
-func MustNewSteps(v *viper.Viper) *Steps {
+func NewSteps(ctx context.Context, v *viper.Viper) (*Steps, error) {
 	v.AutomaticEnv()
 	v.SetEnvPrefix("ZITADEL")
 	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	v.SetConfigType("yaml")
 	err := v.ReadConfig(bytes.NewBuffer(defaultSteps))
-	logging.OnError(err).Fatal("unable to read setup steps")
+	if err != nil {
+		return nil, fmt.Errorf("unable to read default steps: %w", err)
+	}
 
 	for _, file := range stepFiles {
 		v.SetConfigFile(file)
 		err := v.MergeInConfig()
-		logging.WithFields("file", file).OnError(err).Warn("unable to read setup file")
+		logging.OnError(ctx, err).Warn("unable to read setup file", "file", file)
 	}
 
 	steps := new(Steps)
@@ -205,6 +211,8 @@ func MustNewSteps(v *viper.Viper) *Steps {
 			mapstructure.TextUnmarshallerHookFunc(),
 		)),
 	)
-	logging.OnError(err).Fatal("unable to read steps")
-	return steps
+	if err != nil {
+		return nil, fmt.Errorf("unable to read steps: %w", err)
+	}
+	return steps, nil
 }

cmd/setup/config_test.go

@@ -8,6 +8,7 @@ import (
 	"testing"
 
 	"github.com/muhlemmer/gu"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -238,10 +239,13 @@ Actions:
 	}}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			c := &cobra.Command{}
+			c.SetContext(t.Context())
+
 			v := viper.New()
 			v.SetConfigType("yaml")
 			require.NoError(t, v.ReadConfig(strings.NewReader(tt.args.yaml)))
-			got, _, err := NewConfig(t.Context(), v)
+			got, _, err := NewConfig(c, v)
 			require.NoError(t, err)
 			tt.want(t, got)
 		})

cmd/setup/setup.go

@@ -6,7 +6,6 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
-	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
@@ -17,8 +16,8 @@ import (
 	"github.com/jackc/pgx/v5/pgconn"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/build"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/key"
@@ -64,9 +63,7 @@ Requirements:
 - postgreSQL`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel setup command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel setup command failed")
 			}()
 
 			err = tls.ModeFromFlag(cmd)
@@ -84,7 +81,7 @@ Requirements:
 				return fmt.Errorf("unable to bind \"for-mirror\" flag: %w", err)
 			}
 
-			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -92,15 +89,16 @@ Requirements:
 				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
 
-			steps := MustNewSteps(viper.New())
+			steps, err := NewSteps(cmd.Context(), viper.New())
+			if err != nil {
+				return err
+			}
 
 			masterKey, err := key.MasterKey(cmd)
 			if err != nil {
 				return fmt.Errorf("no master key provided: %w", err)
 			}
-
-			Setup(cmd.Context(), config, steps, masterKey)
-			return nil
+			return Setup(cmd.Context(), config, steps, masterKey)
 		},
 	}
 
@@ -127,9 +125,8 @@ func bindForMirror(cmd *cobra.Command) error {
 	return viper.BindPFlag("ForMirror", cmd.Flags().Lookup("for-mirror"))
 }
 
-func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) {
-	logging.Info("setup started")
-
+func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) (err error) {
+	logging.Info(ctx, "setup started")
 	var setupErr error
 	ctx, stop := signal.NotifyContext(ctx, os.Interrupt, syscall.SIGTERM)
 
@@ -137,15 +134,14 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		stop()
 
 		if setupErr == nil {
-			logging.Info("setup completed")
+			logging.Info(ctx, "setup completed")
 			return
 		}
 
-		if setupErr != nil && !errors.Is(setupErr, context.Canceled) {
+		if !errors.Is(setupErr, context.Canceled) {
 			// If Setup failed for some other reason than the context being cancelled,
 			// then this could be a fatal error we should not retry
-			logging.WithFields("error", setupErr).Fatal("setup failed, skipping cleanup")
-			return
+			logging.OnError(ctx, setupErr).Fatal("setup failed, skipping cleanup")
 		}
 
 		// if we're in the middle of long-running setup, run cleanup before exiting
@@ -155,12 +151,13 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 		cleanupCtx, cleanupCancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
 		defer cleanupCancel()
 
-		Cleanup(cleanupCtx, config)
+		err = Cleanup(cleanupCtx, config)
+		logging.OnError(ctx, err).Error("setup cleanup failed")
 	}()
 
 	i18n.MustLoadSupportedLanguagesFromDir()
 	dbClient, err := database.Connect(config.Database, false)
-	logging.OnError(err).Fatal("unable to connect to database")
+	logging.OnError(ctx, err).Fatal("unable to connect to database")
 
 	config.Eventstore.Querier = old_es.NewPostgres(dbClient)
 	esV3 := new_es.NewEventstore(dbClient)
@@ -168,7 +165,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 	config.Eventstore.Searcher = esV3
 	eventstoreClient := eventstore.NewEventstore(config.Eventstore)
 
-	logging.OnError(err).Fatal("unable to start eventstore")
+	logging.OnError(ctx, err).Fatal("unable to start eventstore")
 	eventstoreV4 := es_v4.NewEventstoreFromOne(es_v4_pg.New(dbClient, &es_v4_pg.Config{
 		MaxRetries: config.Eventstore.MaxRetries,
 	}))
@@ -252,7 +249,9 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 	steps.s69CacheTablesLogged = &CacheTablesLogged{dbClient: dbClient}
 
 	err = projection.Create(ctx, dbClient, eventstoreClient, config.Projections, nil, nil, nil)
-	logging.OnError(err).Fatal("unable to start projections")
+	if err != nil {
+		return fmt.Errorf("unable to create projections: %w", err)
+	}
 
 	for _, step := range []migration.Migration{
 		steps.s14NewEventsTable,
@@ -383,6 +382,7 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string)
 			return
 		}
 	}
+	return nil
 }
 
 func executeMigration(ctx context.Context, eventstoreClient *eventstore.Eventstore, step migration.Migration, errorMsg string) error {
@@ -403,8 +403,8 @@ func executeMigration(ctx context.Context, eventstoreClient *eventstore.Eventsto
 			"hint", pgErr.Hint,
 		)
 	}
-	logging.WithFields(logFields...).WithError(err).Error(errorMsg)
-	return fmt.Errorf("%s: %w", errorMsg, err)
+	logging.WithError(ctx, err).Error(errorMsg, logFields...)
+	return err
 }
 
 // readStmt reads a single file from the embedded FS,
@@ -455,10 +455,10 @@ func startCommandsQueries(
 	*auth_view.View,
 ) {
 	keyStorage, err := cryptoDB.NewKeyStorage(dbClient, masterKey)
-	logging.OnError(err).Fatal("unable to start key storage")
+	logging.OnError(ctx, err).Fatal("unable to start key storage")
 
 	keys, err := encryption.EnsureEncryptionKeys(ctx, config.EncryptionKeys, keyStorage)
-	logging.OnError(err).Fatal("unable to ensure encryption keys")
+	logging.OnError(ctx, err).Fatal("unable to ensure encryption keys")
 
 	err = projection.Create(
 		ctx,
@@ -473,13 +473,13 @@ func startCommandsQueries(
 		keys.SAML,
 		config.SystemAPIUsers,
 	)
-	logging.OnError(err).Fatal("unable to start projections")
+	logging.OnError(ctx, err).Fatal("unable to start projections")
 
 	staticStorage, err := config.AssetStorage.NewStorage(dbClient.DB)
-	logging.OnError(err).Fatal("unable to start asset storage")
+	logging.OnError(ctx, err).Fatal("unable to start asset storage")
 
 	adminView, err := admin_view.StartView(dbClient)
-	logging.OnError(err).Fatal("unable to start admin view")
+	logging.OnError(ctx, err).Fatal("unable to start admin view")
 	admin_handler.Register(ctx,
 		admin_handler.Config{
 			Client:                dbClient,
@@ -494,7 +494,7 @@ func startCommandsQueries(
 	sessionTokenVerifier := internal_authz.SessionTokenVerifier(keys.OIDC)
 
 	cacheConnectors, err := connector.StartConnectors(config.Caches, dbClient)
-	logging.OnError(err).Fatal("unable to start caches")
+	logging.OnError(ctx, err).Fatal("unable to start caches")
 
 	queries, err := query.StartQueries(
 		ctx,
@@ -523,10 +523,10 @@ func startCommandsQueries(
 		nil, // not needed for projections
 		false,
 	)
-	logging.OnError(err).Fatal("unable to start queries")
+	logging.OnError(ctx, err).Fatal("unable to start queries")
 
 	authView, err := auth_view.StartView(dbClient, keys.OIDC, queries, eventstoreClient)
-	logging.OnError(err).Fatal("unable to start admin view")
+	logging.OnError(ctx, err).Fatal("unable to start auth view")
 	auth_handler.Register(ctx,
 		auth_handler.Config{
 			Client:                dbClient,
@@ -539,7 +539,7 @@ func startCommandsQueries(
 	)
 
 	authZRepo, err := authz.Start(queries, eventstoreClient, dbClient, keys.OIDC, config.ExternalSecure)
-	logging.OnError(err).Fatal("unable to start authz repo")
+	logging.OnError(ctx, err).Fatal("unable to start authz repo")
 	permissionCheck := func(ctx context.Context, permission, orgID, resourceID string) (err error) {
 		return internal_authz.CheckPermission(ctx, authZRepo, config.SystemAuthZ.RolePermissionMappings, config.InternalAuthZ.RolePermissionMappings, permission, orgID, resourceID)
 	}
@@ -577,12 +577,12 @@ func startCommandsQueries(
 		nil,
 		nil,
 	)
-	logging.OnError(err).Fatal("unable to start commands")
+	logging.OnError(ctx, err).Fatal("unable to start commands")
 
 	q, err := queue.NewQueue(&queue.Config{
 		Client: dbClient,
 	})
-	logging.OnError(err).Fatal("unable to init queue")
+	logging.OnError(ctx, err).Fatal("unable to init queue")
 
 	notify_handler.Register(
 		ctx,
@@ -616,28 +616,28 @@ func initProjections(
 ) error {
 	for _, p := range projection.Projections() {
 		if err := migration.Migrate(ctx, eventstoreClient, p); err != nil {
-			logging.WithFields("name", p.String()).OnError(err).Error("projection migration failed")
+			logging.WithError(ctx, err).Error("projection migration failed", "name", p.String())
 			return err
 		}
 	}
 
 	for _, p := range admin_handler.Projections() {
 		if err := migration.Migrate(ctx, eventstoreClient, p); err != nil {
-			logging.WithFields("name", p.String()).OnError(err).Error("admin schema migration failed")
+			logging.WithError(ctx, err).Error("admin schema migration failed", "name", p.String())
 			return err
 		}
 	}
 
 	for _, p := range auth_handler.Projections() {
 		if err := migration.Migrate(ctx, eventstoreClient, p); err != nil {
-			logging.WithFields("name", p.String()).OnError(err).Error("auth schema migration failed")
+			logging.WithError(ctx, err).Error("auth schema migration failed", "name", p.String())
 			return err
 		}
 	}
 
 	for _, p := range notify_handler.Projections() {
 		if err := migration.Migrate(ctx, eventstoreClient, p); err != nil {
-			logging.WithFields("name", p.String()).OnError(err).Error("notification migration failed")
+			logging.WithError(ctx, err).Error("notification migration failed", "name", p.String())
 			return err
 		}
 	}

cmd/setup/steps.yaml

@@ -12,11 +12,11 @@ Instrumentation:
       # "stdErr": Exports traces to standard error
       # "grpc": Exports traces using the OTEL gRPC exporter (recommended)
       # "http": Exports traces using the OTEL HTTP exporter
-      # "google": Exports traces to Google Cloud Trace
+      # "google": Exports traces to Google Cloud. Need to configure GoogleProjectID below!
       Type: "none" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_TYPE
       # Endpoint of the OTEL collector for grpc and http exporters
       Endpoint: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_ENDPOINT
-      # Disable https for grpc and http exporters
+      # Disable TLS for grpc and http exporters
       Insecure: false # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_INSECURE
       # Interval for batching traces before export
       BatchDuration: 1s # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_BATCHDURATION
@@ -30,15 +30,15 @@ Instrumentation:
       # "stdErr": Exports metrics to standard error
       # "grpc": Exports metrics using the OTEL gRPC exporter (recommended)
       # "http": Exports metrics using the OTEL HTTP exporter
-      # "google": Exports metrics to Google Cloud Trace
+      # "google": Exports metrics to Google Cloud. Need to configure GoogleProjectID below!
       # "prometheus": Exposes metrics via an HTTP endpoint for Prometheus to scrape
       Type: "none" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_TYPE
       # Endpoint of the OTEL collector for grpc and http exporters
       Endpoint: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_ENDPOINT
-      # Disable https for grpc and http exporters
+      # Disable TLS for grpc and http exporters
       Insecure: false # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_INSECURE
       # Interval at which metrics are exported
-      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_BATCHDURATION
+      BatchDuration: 1m # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_BATCHDURATION
       # Project ID for Google Cloud Trace exporter
       GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_GOOGLEPROJECTID
   
@@ -46,39 +46,67 @@ Instrumentation:
   Log:
     # Log lines lower than this level are not emitted.
     Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Streams enable logging for specific parts of the application.
+    Streams: # ZITADEL_INSTRUMENTATION_LOG_STREAMS (comma separated list)
+      - runtime # General runtime logs, such as startup and shutdown messages.
+      - request # Logs for incoming API and HTTP requests.
+      - event_handler # Logs for event handling in projections.
+      - queue # Logs for the job queue processing.
+      # - event_pusher # Logs for event pushing to the database. Warning: contains sensitive information.
+    # Mask replaces sensitive information with Value in logs matched by Key
+    Mask:
+      # Keys are the attribute keys to be masked in logs.
+      # Keys are unqualified attribute names and apply to all attributes with the specified name,
+      # regardless of their position in the attribute hierarchy.
+      # Eg. "some_key" matches "some_key" and "parent.some_key" etc.
+      # If the matched log attribute is a nested object or array,
+      # the entire structure is replaced with the specified Value.
+      Keys: # ZITADEL_INSTRUMENTATION_LOG_MASK_KEYS (comma separated list)
+        # - "first_name"
+        # - "last_name"
+      # Value is the string that replaces the original value of masked attributes.
+      Value: "****" # ZITADEL_INSTRUMENTATION_LOG_MASK_VALUE
     # Enable printing structured logs to standard error in the specified format.
+    # When disabled, the legacy Log configuration is used.
+    # Important: when legacy Log is customized, please add a format and customize before upgrading to v5
     # The following formats are supported:
-    #   - "disabled": Disables logging to standard error
-    #   - "text": Logs are printed as human-readable text
-    #   - "json": Logs are printed as JSON objects
-    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging
-    StdErr: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    #   - "disabled": Disables logging (or fallback to legacy Log config)
+    #   - "text": Logs are printed on StdErr as human-readable text
+    #   - "json": Logs are printed on StdErr as JSON objects
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging.
+    #   - "gcp_error_reporting": JSON formatted logs compatible with Google Cloud Platform Error Reporting.
+    Format: "disabled" # ZITADEL_INSTRUMENTATION_LOG_STDERR
     # Adds the source file and line number where the log line was emitted.
     # Not to be confused with the source of an error.
     AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
+    # Configure how errors are logged.
+    Errors:
+      # Adds the source file, line number and function where the error was created.
+      ReportLocation: true # ZITADEL_INSTRUMENTATION_LOG_ERRORS_REPORTLOCATION
+      # Adds stack traces to logged errors.
+      StackTrace: false # ZITADEL_INSTRUMENTATION_LOG_ERRORS_STACKTRACE
     Exporter:
       # The following exporter types are supported:
-      #   - "none": Disables log exporting
-      #   - "stdOut": Exports logs to standard output
-      #   - "stdErr": Exports logs to standard error
-      #   - "grpc": Exports logs using the OTEL gRPC exporter (recommended)
-      #   - "http": Exports logs using the OTEL HTTP exporter
+      #   - "none": Disables OTEL log exporter
+      #   - "stdOut": Exports OTEL logs to standard output
+      #   - "stdErr": Exports OTEL logs to standard error
+      #   - "grpc": Exports OTEL logs using the OTEL gRPC exporter (recommended)
+      #   - "http": Exports OTEL logs using the OTEL HTTP exporter
       Type: "none" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_TYPE
       # Endpoint of the OTEL collector for grpc and http exporters
       Endpoint: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_ENDPOINT
-      # Disable https for grpc and http exporters
+      # Disable TLS for grpc and http exporters
       Insecure: false # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_INSECURE
       # Interval at which metrics are exported
       BatchDuration: 1s # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_BATCHDURATION
-      # Project ID for Google Cloud Trace exporter
-      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_GOOGLEPROJECTID
   Profile:
-    # The following profiler types are supported:
-    # "none": Disables profiling
-    # "google": Exports profiling data to Google Cloud Profiler
-    Type: "none" # ZITADEL_INSTRUMENTATION_PROFILE_TYPE
-    # Project ID for Google Cloud Profiler
-    GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_PROFILE_GOOGLEPROJECTID
+    Exporter:
+      # The following profiler types are supported:
+      # "none": Disables profiling
+      # "google": Exports profiling data to Google Cloud Profiler
+      Type: "none" # ZITADEL_INSTRUMENTATION_PROFILE_TYPE
+      # Project ID for Google Cloud Profiler
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_PROFILE_GOOGLEPROJECTID
 
 # By using the FirstInstance section, you can overwrite the DefaultInstance configuration for the first instance created by zitadel setup.
 FirstInstance:

cmd/setup/sync_role_permissions.go

@@ -5,8 +5,7 @@ import (
 	_ "embed"
 	"fmt"
 
-	"github.com/zitadel/logging"
-
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/internal/api/authz"
 	"github.com/zitadel/zitadel/internal/command"
 	"github.com/zitadel/zitadel/internal/eventstore"
@@ -23,6 +22,7 @@ type SyncRolePermissions struct {
 }
 
 func (mig *SyncRolePermissions) Execute(ctx context.Context, _ eventstore.Event) error {
+	ctx = logging.With(ctx, "migration", mig.String())
 	if err := mig.executeSystem(ctx); err != nil {
 		return err
 	}
@@ -30,12 +30,12 @@ func (mig *SyncRolePermissions) Execute(ctx context.Context, _ eventstore.Event)
 }
 
 func (mig *SyncRolePermissions) executeSystem(ctx context.Context) error {
-	logging.WithFields("migration", mig.String()).Info("prepare system role permission sync events")
+	logging.Info(ctx, "prepare system role permission sync events")
 	details, err := mig.commands.SynchronizeRolePermission(ctx, "SYSTEM", mig.rolePermissionMappings)
 	if err != nil {
 		return err
 	}
-	logging.WithFields("migration", mig.String(), "sequence", details.Sequence).Info("pushed system role permission sync events")
+	logging.Info(ctx, "pushed system role permission sync events", "sequence", details.Sequence)
 	return nil
 }
 
@@ -57,12 +57,12 @@ func (mig *SyncRolePermissions) executeInstances(ctx context.Context) error {
 		return err
 	}
 	for i, instanceID := range instances {
-		logging.WithFields("instance_id", instanceID, "migration", mig.String(), "progress", fmt.Sprintf("%d/%d", i+1, len(instances))).Info("prepare instance role permission sync events")
+		logging.Info(ctx, "prepare instance role permission sync events", "instance", instanceID, "progress", fmt.Sprintf("%d/%d", i+1, len(instances)))
 		details, err := mig.commands.SynchronizeRolePermission(ctx, instanceID, mig.rolePermissionMappings)
 		if err != nil {
 			return err
 		}
-		logging.WithFields("instance_id", instanceID, "migration", mig.String(), "sequence", details.Sequence).Info("pushed instance role permission sync events")
+		logging.Info(ctx, "pushed instance role permission sync events", "instance", instanceID, "sequence", details.Sequence)
 	}
 	return nil
 }

cmd/start/config.go

@@ -1,16 +1,17 @@
 package start
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
+	old_logging "github.com/zitadel/logging" //nolint:staticcheck
 
 	"github.com/zitadel/zitadel/backend/v3/instrumentation"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -42,7 +43,7 @@ import (
 
 type Config struct {
 	Instrumentation     instrumentation.Config
-	Log                 *logging.Config
+	Log                 *old_logging.Config
 	Port                uint16
 	ExternalPort        uint16
 	ExternalDomain      string
@@ -95,7 +96,7 @@ type QuotasConfig struct {
 	Execution *logstore.EmitterConfig
 }
 
-func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
+func NewConfig(cmd *cobra.Command, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config, err := readConfig(v)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to read config: %w", err)
@@ -103,16 +104,18 @@ func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.Sh
 
 	config.Instrumentation.Trace.SetLegacyConfig(config.Tracing)
 	config.Instrumentation.Metric.SetLegacyConfig(config.Metrics)
+	config.Instrumentation.Log.SetLegacyConfig(config.Log)
 	config.Instrumentation.Profile.SetLegacyConfig(config.Profiler)
-	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	shutdown, err := instrumentation.Start(cmd.Context(), config.Instrumentation)
 	if err != nil {
 		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
 	}
+	cmd.SetContext(logging.NewCtx(cmd.Context(), logging.StreamRuntime))
 
 	// Legacy logger
 	err = config.Log.SetLogger()
 	if err != nil {
-		err = errors.Join(err, shutdown(ctx))
+		err = errors.Join(err, shutdown(cmd.Context()))
 		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
 	}
 
@@ -123,7 +126,7 @@ func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.Sh
 
 	err = config.SystemDefaults.Validate()
 	if err != nil {
-		err = errors.Join(err, shutdown(ctx))
+		err = errors.Join(err, shutdown(cmd.Context()))
 		return nil, nil, fmt.Errorf("system defaults config invalid: %w", err)
 	}
 	// Copy the global role permissions mappings to the instance until we allow instance-level configuration over the API.

cmd/start/flags.go

@@ -4,8 +4,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/tls"
 )
@@ -23,6 +23,7 @@ func init() {
 func startFlags(cmd *cobra.Command) {
 	cmd.Flags().AddFlagSet(startFlagSet)
 	logging.OnError(
+		cmd.Context(),
 		viper.BindPFlags(startFlagSet),
 	).Fatal("start flags")
 

cmd/start/start.go

@@ -6,7 +6,6 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
-	"log/slog"
 	"math"
 	"net/http"
 	"os"
@@ -21,7 +20,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 	"github.com/zitadel/oidc/v3/pkg/op"
 	"github.com/zitadel/saml/pkg/provider"
 	"golang.org/x/net/http2"
@@ -29,6 +27,7 @@ import (
 	"golang.org/x/text/language"
 
 	new_domain "github.com/zitadel/zitadel/backend/v3/domain"
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	v3_postgres "github.com/zitadel/zitadel/backend/v3/storage/database/dialect/postgres"
 	"github.com/zitadel/zitadel/cmd/build"
 	"github.com/zitadel/zitadel/cmd/encryption"
@@ -131,16 +130,14 @@ Requirements:
 - postgreSQL`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel start command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel start command failed")
 			}()
 
 			err = cmd_tls.ModeFromFlag(cmd)
 			if err != nil {
 				return err
 			}
-			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			config, shutdown, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -342,6 +339,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 	notification.Start(ctx)
 
 	execution.Register(
+		ctx,
 		config.Executions,
 		q,
 		keys.Target,
@@ -359,7 +357,7 @@ func startZitadel(ctx context.Context, config *Config, masterKey string, server
 	}
 
 	// the scheduler / periodic jobs need to be started after the queue already runs
-	if err = serviceping.Start(config.ServicePing, q); err != nil {
+	if err = serviceping.Start(ctx, config.ServicePing, q); err != nil {
 		return err
 	}
 
@@ -750,7 +748,7 @@ func listen(ctx context.Context, router *mux.Router, port uint16, tlsConfig *tls
 	errCh := make(chan error)
 
 	go func() {
-		logging.Infof("server is listening on %s", lis.Addr().String())
+		logging.Info(ctx, "server is listening", "address", lis.Addr().String())
 		if tlsConfig != nil {
 			// we don't need to pass the files here, because we already initialized the TLS config on the server
 			errCh <- http1Server.ServeTLS(lis, "", "")
@@ -776,7 +774,7 @@ func shutdownServer(ctx context.Context, server *http.Server) error {
 	if err != nil {
 		return fmt.Errorf("could not shutdown gracefully: %w", err)
 	}
-	logging.New().Info("server shutdown gracefully")
+	logging.Info(ctx, "server shutdown gracefully")
 	return nil
 }
 

cmd/start/start_from_init.go

@@ -4,11 +4,11 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/initialise"
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/setup"
@@ -28,9 +28,7 @@ Requirements:
 - postgreSQL`,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel start-from-init command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel start-from-init command failed")
 			}()
 
 			err = tls.ModeFromFlag(cmd)
@@ -43,30 +41,42 @@ Requirements:
 				return fmt.Errorf("no master key provided: %w", err)
 			}
 
-			initCtx, cancel := context.WithCancel(cmd.Context())
-			initialise.InitAll(initCtx, initialise.MustNewConfig(viper.GetViper()))
-			cancel()
-
-			err = setup.BindInitProjections(cmd)
-			if err != nil {
-				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
-			}
-
-			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			initConfig, shutdown, err := initialise.NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
 			defer func() {
 				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
+			initCtx, cancel := context.WithCancel(cmd.Context())
+			defer cancel()
 
-			setupSteps := setup.MustNewSteps(viper.New())
+			err = initialise.InitAll(initCtx, initConfig)
+			if err != nil {
+				return err
+			}
 
-			setupCtx, cancel := context.WithCancel(cmd.Context())
-			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
-			cancel()
+			err = setup.BindInitProjections(cmd)
+			if err != nil {
+				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
+			}
 
-			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			setupConfig, _, err := setup.NewConfig(cmd, viper.GetViper())
+			if err != nil {
+				return err
+			}
+
+			setupSteps, err := setup.NewSteps(cmd.Context(), viper.New())
+			if err != nil {
+				return err
+			}
+
+			err = setup.Setup(cmd.Context(), setupConfig, setupSteps, masterKey)
+			if err != nil {
+				return err
+			}
+
+			startConfig, _, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}

cmd/start/start_from_setup.go

@@ -1,13 +1,12 @@
 package start
 
 import (
-	"context"
 	"errors"
-	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/setup"
 	"github.com/zitadel/zitadel/cmd/tls"
@@ -27,9 +26,7 @@ Requirements:
 `,
 		RunE: func(cmd *cobra.Command, args []string) (err error) {
 			defer func() {
-				if err != nil {
-					slog.Error("zitadel start-from-setup command failed", "err", err)
-				}
+				logging.OnError(cmd.Context(), err).Error("zitadel start-from-setup command failed")
 			}()
 
 			err = tls.ModeFromFlag(cmd)
@@ -47,7 +44,7 @@ Requirements:
 				return err
 			}
 
-			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			setupConfig, shutdown, err := setup.NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}
@@ -55,13 +52,17 @@ Requirements:
 				err = errors.Join(err, shutdown(cmd.Context()))
 			}()
 
-			setupSteps := setup.MustNewSteps(viper.New())
+			setupSteps, err := setup.NewSteps(cmd.Context(), viper.New())
+			if err != nil {
+				return err
+			}
 
-			setupCtx, cancel := context.WithCancel(cmd.Context())
-			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
-			cancel()
+			err = setup.Setup(cmd.Context(), setupConfig, setupSteps, masterKey)
+			if err != nil {
+				return err
+			}
 
-			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			startConfig, _, err := NewConfig(cmd, viper.GetViper())
 			if err != nil {
 				return err
 			}

cmd/zitadel.go

@@ -2,6 +2,7 @@ package cmd
 
 import (
 	"bytes"
+	"context"
 	_ "embed"
 	"errors"
 	"io"
@@ -9,8 +10,8 @@ import (
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation/logging"
 	"github.com/zitadel/zitadel/cmd/admin"
 	"github.com/zitadel/zitadel/cmd/build"
 	"github.com/zitadel/zitadel/cmd/initialise"
@@ -45,7 +46,7 @@ func New(out io.Writer, in io.Reader, args []string, server chan<- *start.Server
 	viper.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	viper.SetConfigType("yaml")
 	err := viper.ReadConfig(bytes.NewBuffer(defaultConfig))
-	logging.OnError(err).Fatal("unable to read default config")
+	logging.OnError(context.Background(), err).Fatal("unable to read default config")
 
 	cobra.OnInitialize(initConfig)
 	cmd.PersistentFlags().StringArrayVar(&configFiles, "config", nil, "path to config file to overwrite system defaults")
@@ -71,6 +72,6 @@ func initConfig() {
 	for _, file := range configFiles {
 		viper.SetConfigFile(file)
 		err := viper.MergeInConfig()
-		logging.WithFields("file", file).OnError(err).Warn("unable to read config file")
+		logging.OnError(context.Background(), err).Warn("unable to read config file", "file", file)
 	}
 }