feat(telemetry): improved instrumentation for observability (#11159)

# Which Problems Are Solved

Zitadel did not provide easy correlation between errors, logs, traces
and metrics. The configuration for those instrumentations was also not
consistent, with some supporting different exporters then others.
Implementation and parsing of config was also spaghettified over
multiple packages, with awkward parsing and inconsistent naming of
options.

# How the Problems Are Solved

All telemetry is now merged under the name "instrumentation". Why?
1. We thought it was a good idea in the past to call the milestone
exporter `Telemtry` in the runtime config. Calling this `TelemetryV2`
looks weird.
2. Not everything is a meter and not everything is sent (tele...). 
3. It's also
[defined](https://opentelemetry.io/docs/concepts/instrumentation/) as
such by the OTEL documentation.

## New features

- Adds structured, context based logging with trace-ID awareness
- Static log fields are added to the context, such as service and
request path
- Static log fields are injected in each logline emitted by the
application
- Structured logs can also be send to an otel exporter
- Structured logs can be printed to StdErr in text and JSON format
- Error sinks make sure every error is logged at the correct level:
- Warnings for client side errors (HTTP 400 range, Invalid request etc)
  - Error for server side errors (Internal server errors)
- Metrics can now also be send to a OTEL collector. (previously they
could only be scraped from `/debug/metrics` with prometheus)

## Exporters

This change adds all the exporters supported by OTEL upstream and some
google specific exporters for our cloud deployment.

- StdOut / StdErr: all instrumentations
- OTEL gRPC / HTTP: all instrumentations
- Google: all instrumentations except logging
- Prometheus (pull-based): only metrics

The exception is profiling, which only supports the google exporting due
to lack of support by OTEL upstream.

## Configuration and structure 

- All instrumentation is moved into the new `backend/v3/instrumentation`
package. It reuses configuration types, so both code and runtime
configuration are easier to understand.
- The `internal/telemetry` packages are removed.
- Instrumentation is started with a single function and a proper
shutdown function is now provided.
- Legacy configuration is still parsed from the runtime config, as long
as the new configuration is disabled. This allows backporting this
feature to v4 without breaking existing configurations.

# Additional Changes

- Devcontainer: set `$PATH` variable so installed go binaries can be run
individually, without NX.
- NX: install GCI tool to fix imports

# Additional Context

- Closes https://github.com/zitadel/zitadel/issues/8408
- Closes https://github.com/zitadel/zitadel/issues/6664
- Backport to v4

cmd/admin/admin.go

@@ -3,6 +3,7 @@ package admin
 import (
 	_ "embed"
 	"errors"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 
@@ -18,7 +19,12 @@ func New() *cobra.Command {
 		Short:      "The ZITADEL admin CLI lets you interact with your instance",
 		Long:       `The ZITADEL admin CLI lets you interact with your instance`,
 		Deprecated: "please use subcommands directly, e.g. `zitadel start`",
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel admin command failed", "err", err)
+				}
+			}()
 			return errors.New("no additional command provided")
 		},
 	}

cmd/defaults.yaml

@@ -1,13 +1,94 @@
+Instrumentation:
+  ServiceName: "zitadel" # ZITADEL_INSTRUMENTATION_SERVICENAME
+  Trace:
+    Fraction: 1.0 # ZITADEL_INSTRUMENTATION_TRACE_FRACTION
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables trace exporting
+      # "stdOut": Exports traces to standard output
+      # "stdErr": Exports traces to standard error
+      # "grpc": Exports traces using the OTEL gRPC exporter (recommended)
+      # "http": Exports traces using the OTEL HTTP exporter
+      # "google": Exports traces to Google Cloud. Need to configure GoogleProjectID below!
+      Type: "none" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_INSECURE
+      # Interval for batching traces before export
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_GOOGLEPROJECTID
+  Metric:
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables metric exporting
+      # "stdOut": Exports metrics to standard output
+      # "stdErr": Exports metrics to standard error
+      # "grpc": Exports metrics using the OTEL gRPC exporter (recommended)
+      # "http": Exports metrics using the OTEL HTTP exporter
+      # "google": Exports metrics to Google Cloud. Need to configure GoogleProjectID below!
+      # "prometheus": Exposes metrics via an HTTP endpoint for Prometheus to scrape
+      Type: "none" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1m # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_GOOGLEPROJECTID
+  
+  # Structured logging can print logs on standard error and/or export them using OTEL.
+  Log:
+    # Log lines lower than this level are not emitted.
+    Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Enable printing structured logs to standard error in the specified format.
+    # The following formats are supported:
+    #   - "disabled": Disables logging
+    #   - "text": Logs are printed on StdErr as human-readable text
+    #   - "json": Logs are printed on StdErr as JSON objects
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging. Need to configure GoogleProjectID below!
+    Format: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    # Adds the source file and line number where the log line was emitted.
+    # Not to be confused with the source of an error.
+    AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
+    Exporter:
+      # The following exporter types are supported:
+      #   - "none": Disables OTEL log exporter
+      #   - "stdOut": Exports OTEL logs to standard output
+      #   - "stdErr": Exports OTEL logs to standard error
+      #   - "grpc": Exports OTEL logs using the OTEL gRPC exporter (recommended)
+      #   - "http": Exports OTEL logs using the OTEL HTTP exporter
+      Type: "none" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_BATCHDURATION
+  Profile:
+    Exporter:
+      # The following profiler types are supported:
+      # "none": Disables profiling
+      # "google": Exports profiling data to Google Cloud Profiler
+      Type: "none" # ZITADEL_INSTRUMENTATION_PROFILE_TYPE
+      # Project ID for Google Cloud Profiler
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_PROFILE_GOOGLEPROJECTID
+
+# Legacy logging (Deprecated, see Instrumentation.Slog)
 Log:
   Level: info # ZITADEL_LOG_LEVEL
   Formatter:
     Format: text # ZITADEL_LOG_FORMATTER_FORMAT
 
-# Exposes metrics on /debug/metrics
+# Exposes metrics on /debug/metrics (Deprecated, see Instrumentation.Metric.Prometheus)
 Metrics:
   # Select type otel (OpenTelemetry) or none (disables collection and endpoint)
   Type: otel # ZITADEL_METRICS_TYPE
 
+
+# Collects and exports tracing information (Deprecated, see Instrumentation.Trace)
 Tracing:
   # Choose one in "otel", "google", "log" and "none"
   # Depending on the type there are different configuration options

cmd/initialise/init.go

@@ -3,6 +3,8 @@ package initialise
 import (
 	"context"
 	"embed"
+	"fmt"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -43,10 +45,15 @@ The user provided by flags needs privileges to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel init command failed", "err", err)
+				}
+			}()
 			config := MustNewConfig(viper.GetViper())
 
-			InitAll(cmd.Context(), config)
+			return InitAll(cmd.Context(), config)
 		},
 	}
 
@@ -54,16 +61,21 @@ The user provided by flags needs privileges to
 	return cmd
 }
 
-func InitAll(ctx context.Context, config *Config) {
+func InitAll(ctx context.Context, config *Config) error {
 	err := initialise(ctx, config.Database,
 		VerifyUser(config.Database.Username(), config.Database.Password()),
 		VerifyDatabase(config.Database.DatabaseName()),
 		VerifyGrant(config.Database.DatabaseName(), config.Database.Username()),
 	)
-	logging.OnError(err).Fatal("unable to initialize the database")
+	if err != nil {
+		return fmt.Errorf("initialize database failed: %w", err)
+	}
 
 	err = verifyZitadel(ctx, config.Database)
-	logging.OnError(err).Fatal("unable to initialize ZITADEL")
+	if err != nil {
+		return fmt.Errorf("initialize ZITADEL failed: %w", err)
+	}
+	return nil
 }
 
 func initialise(ctx context.Context, config database.Config, steps ...func(context.Context, *database.DB) error) error {

cmd/key/key.go

@@ -2,6 +2,7 @@ package key
 
 import (
 	"io"
+	"log/slog"
 	"os"
 	"strings"
 
@@ -44,7 +45,13 @@ Requirements:
 		Example: `new -f keys.yaml
 new key1=somekey key2=anotherkey
 new -f keys.yaml key2=anotherkey`,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel keys new command failed", "err", err)
+				}
+			}()
+
 			keys, err := keysFromArgs(args)
 			if err != nil {
 				return err

cmd/mirror/auth.go

@@ -3,6 +3,7 @@ package mirror
 import (
 	"context"
 	_ "embed"
+	"errors"
 	"io"
 	"strconv"
 	"time"
@@ -22,9 +23,16 @@ func authCmd() *cobra.Command {
 		Long: `mirrors the auth requests table from one database to another
 ZITADEL needs to be initialized and set up with the --for-mirror flag
 Only auth requests are mirrored`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			copyAuth(cmd.Context(), config)
+			return nil
 		},
 	}
 

cmd/mirror/config.go

@@ -1,13 +1,16 @@
 package mirror
 
 import (
+	"context"
 	_ "embed"
+	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
 	internal_authz "github.com/zitadel/zitadel/internal/api/authz"
@@ -16,7 +19,6 @@ import (
 	"github.com/zitadel/zitadel/internal/database"
 	"github.com/zitadel/zitadel/internal/domain"
 	"github.com/zitadel/zitadel/internal/id"
-	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
 )
 
 type Migration struct {
@@ -26,9 +28,10 @@ type Migration struct {
 	EventBulkSize     uint32
 	MaxAuthRequestAge time.Duration
 
-	Log     *logging.Config
-	Machine *id.Config
-	Metrics metrics.Config
+	Log             *logging.Config
+	Machine         *id.Config
+	Instrumentation instrumentation.Config
+	Metrics         instrumentation.LegacyMetricConfig
 }
 
 var (
@@ -36,19 +39,24 @@ var (
 	defaultConfig []byte
 )
 
-func mustNewMigrationConfig(v *viper.Viper) *Migration {
+func mustNewMigrationConfig(ctx context.Context, v *viper.Viper) (*Migration, instrumentation.ShutdownFunc, error) {
 	config := new(Migration)
 	mustNewConfig(v, config)
 
-	err := config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
+	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
 
-	err = config.Metrics.NewMeter()
-	logging.OnError(err).Fatal("unable to set meter")
+	// Legacy logger
+	err = config.Log.SetLogger()
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 
-	return config
+	return config, shutdown, nil
 }
 
 func mustNewProjectionsConfig(v *viper.Viper) *ProjectionsConfig {

cmd/mirror/defaults.yaml

@@ -1,3 +1,81 @@
+Instrumentation:
+  ServiceName: "zitadel-mirror" # ZITADEL_INSTRUMENTATION_SERVICENAME
+  Trace:
+    Fraction: 1.0 # ZITADEL_INSTRUMENTATION_TRACE_FRACTION
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables trace exporting
+      # "stdOut": Exports traces to standard output
+      # "stdErr": Exports traces to standard error
+      # "grpc": Exports traces using the OTEL gRPC exporter (recommended)
+      # "http": Exports traces using the OTEL HTTP exporter
+      # "google": Exports traces to Google Cloud. Need to configure GoogleProjectID below!
+      Type: "none" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_INSECURE
+      # Interval for batching traces before export
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_GOOGLEPROJECTID
+  Metric:
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables metric exporting
+      # "stdOut": Exports metrics to standard output
+      # "stdErr": Exports metrics to standard error
+      # "grpc": Exports metrics using the OTEL gRPC exporter (recommended)
+      # "http": Exports metrics using the OTEL HTTP exporter
+      # "google": Exports metrics to Google Cloud. Need to configure GoogleProjectID below!
+      # "prometheus": Exposes metrics via an HTTP endpoint for Prometheus to scrape
+      Type: "none" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1m # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_GOOGLEPROJECTID
+  
+  # Structured logging can print logs on standard error and/or export them using OTEL.
+  Log:
+    # Log lines lower than this level are not emitted.
+    Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Enable printing structured logs to standard error in the specified format.
+    # The following formats are supported:
+    #   - "disabled": Disables logging
+    #   - "text": Logs are printed on StdErr as human-readable text
+    #   - "json": Logs are printed on StdErr as JSON objects
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging. Need to configure GoogleProjectID below!
+    Format: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    # Adds the source file and line number where the log line was emitted.
+    # Not to be confused with the source of an error.
+    AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
+    Exporter:
+      # The following exporter types are supported:
+      #   - "none": Disables OTEL log exporter
+      #   - "stdOut": Exports OTEL logs to standard output
+      #   - "stdErr": Exports OTEL logs to standard error
+      #   - "grpc": Exports OTEL logs using the OTEL gRPC exporter (recommended)
+      #   - "http": Exports OTEL logs using the OTEL HTTP exporter
+      Type: "none" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_ENDPOINT
+      # Disable TLS for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_BATCHDURATION
+  Profile:
+    Exporter:
+      # The following profiler types are supported:
+      # "none": Disables profiling
+      # "google": Exports profiling data to Google Cloud Profiler
+      Type: "none" # ZITADEL_INSTRUMENTATION_PROFILE_TYPE
+      # Project ID for Google Cloud Profiler
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_PROFILE_GOOGLEPROJECTID
+
 Source:
   cockroach:
     Host: localhost # ZITADEL_SOURCE_COCKROACH_HOST

cmd/mirror/event_store.go

@@ -33,9 +33,16 @@ func eventstoreCmd() *cobra.Command {
 		Long: `mirrors the eventstore of an instance from one database to another
 ZITADEL needs to be initialized and set up with the --for-mirror flag
 Migrate only copies events2 and unique constraints`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			copyEventstore(cmd.Context(), config)
+			return nil
 		},
 	}
 

cmd/mirror/mirror.go

@@ -3,6 +3,9 @@ package mirror
 import (
 	"bytes"
 	_ "embed"
+	"errors"
+	"fmt"
+	"log/slog"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -44,18 +47,34 @@ Order of execution:
 				logging.WithFields("file", file).OnError(err).Warn("unable to read config file")
 			}
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel mirror command failed", "err", err)
+				}
+			}()
+
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return fmt.Errorf("unable to create migration config: %w", err)
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
+
 			projectionConfig := mustNewProjectionsConfig(viper.GetViper())
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Fatal("unable to read master key")
+			if err != nil {
+				return fmt.Errorf("unable to read master key: %w", err)
+			}
 
 			copySystem(cmd.Context(), config)
 			copyAuth(cmd.Context(), config)
 			copyEventstore(cmd.Context(), config)
 
 			projections(cmd.Context(), projectionConfig, masterKey)
+			return nil
 		},
 	}
 

cmd/mirror/system.go

@@ -3,6 +3,7 @@ package mirror
 import (
 	"context"
 	_ "embed"
+	"errors"
 	"io"
 	"time"
 
@@ -21,9 +22,16 @@ func systemCmd() *cobra.Command {
 		Long: `mirrors the system tables of ZITADEL from one database to another
 ZITADEL needs to be initialized
 Only keys and assets are mirrored`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			copySystem(cmd.Context(), config)
+			return nil
 		},
 	}
 

cmd/mirror/verify.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	_ "embed"
+	"errors"
 	"fmt"
 	"slices"
 
@@ -20,9 +21,16 @@ func verifyCmd() *cobra.Command {
 	return &cobra.Command{
 		Use:   "verify",
 		Short: "counts if source and dest have the same amount of entries",
-		Run: func(cmd *cobra.Command, args []string) {
-			config := mustNewMigrationConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := mustNewMigrationConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			verifyMigration(cmd.Context(), config)
+			return nil
 		},
 	}
 }

cmd/setup/cleanup.go

@@ -2,6 +2,7 @@ package setup
 
 import (
 	"context"
+	"errors"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -19,9 +20,16 @@ func NewCleanup() *cobra.Command {
 		Use:   "cleanup",
 		Short: "cleans up migration if they got stuck",
 		Long:  `cleans up migration if they got stuck`,
-		Run: func(cmd *cobra.Command, args []string) {
-			config := MustNewConfig(viper.GetViper())
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 			Cleanup(cmd.Context(), config)
+			return nil
 		},
 	}
 }

cmd/setup/config.go

@@ -2,6 +2,9 @@ package setup
 
 import (
 	"bytes"
+	"context"
+	"errors"
+	"fmt"
 	"strings"
 	"time"
 
@@ -9,6 +12,7 @@ import (
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -27,7 +31,6 @@ import (
 	"github.com/zitadel/zitadel/internal/notification/handlers"
 	"github.com/zitadel/zitadel/internal/query/projection"
 	static_config "github.com/zitadel/zitadel/internal/static/config"
-	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
 )
 
 type Config struct {
@@ -40,8 +43,9 @@ type Config struct {
 	ExternalDomain  string
 	ExternalPort    uint16
 	ExternalSecure  bool
+	Instrumentation instrumentation.Config
 	Log             *logging.Config
-	Metrics         metrics.Config
+	Metrics         *instrumentation.LegacyMetricConfig
 	EncryptionKeys  *encryption.EncryptionKeyConfig
 	DefaultInstance command.InstanceSetup
 	Machine         *id.Config
@@ -66,7 +70,7 @@ type InitProjections struct {
 	BulkLimit        uint64
 }
 
-func MustNewConfig(v *viper.Viper) *Config {
+func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config := new(Config)
 	err := v.Unmarshal(config,
 		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
@@ -85,20 +89,28 @@ func MustNewConfig(v *viper.Viper) *Config {
 			mapstructure.TextUnmarshallerHookFunc(),
 		)),
 	)
-	logging.OnError(err).Fatal("unable to read default config")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to read default config: %w", err)
+	}
+
+	config.Instrumentation.Metric.SetLegacyConfig(config.Metrics)
+	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
 
 	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
-	err = config.Metrics.NewMeter()
-	logging.OnError(err).Fatal("unable to set meter")
+	if err != nil {
+		err = errors.Join(err, shutdown(ctx))
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 
 	// Copy the global role permissions mappings to the instance until we allow instance-level configuration over the API.
 	config.DefaultInstance.RolePermissionMappings = config.InternalAuthZ.RolePermissionMappings
 
-	return config
+	return config, shutdown, nil
 }
 
 type Steps struct {

cmd/setup/config_test.go

@@ -241,7 +241,8 @@ Actions:
 			v := viper.New()
 			v.SetConfigType("yaml")
 			require.NoError(t, v.ReadConfig(strings.NewReader(tt.args.yaml)))
-			got := MustNewConfig(v)
+			got, _, err := NewConfig(t.Context(), v)
+			require.NoError(t, err)
 			tt.want(t, got)
 		})
 	}

cmd/setup/setup.go

@@ -6,6 +6,7 @@ import (
 	_ "embed"
 	"errors"
 	"fmt"
+	"log/slog"
 	"net/http"
 	"os"
 	"os/signal"
@@ -61,23 +62,45 @@ func New() *cobra.Command {
 		Long: `sets up data to start ZITADEL.
 Requirements:
 - postgreSQL`,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel setup command failed", "err", err)
+				}
+			}()
+
+			err = tls.ModeFromFlag(cmd)
+			if err != nil {
+				return fmt.Errorf("invalid tlsMode: %w", err)
+			}
 
 			err = BindInitProjections(cmd)
-			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+			if err != nil {
+				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
+			}
 
 			err = bindForMirror(cmd)
-			logging.OnError(err).Fatal("unable to bind \"for-mirror\" flag")
+			if err != nil {
+				return fmt.Errorf("unable to bind \"for-mirror\" flag: %w", err)
+			}
+
+			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			config := MustNewConfig(viper.GetViper())
 			steps := MustNewSteps(viper.New())
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
+			if err != nil {
+				return fmt.Errorf("no master key provided: %w", err)
+			}
 
 			Setup(cmd.Context(), config, steps, masterKey)
+			return nil
 		},
 	}
 

cmd/setup/steps.yaml

@@ -1,3 +1,82 @@
+Instrumentation:
+  ServiceName: "zitadel-setup" # ZITADEL_INSTRUMENTATION_SERVICENAME
+  Trace:
+    Fraction: 1.0 # ZITADEL_INSTRUMENTATION_TRACE_FRACTION
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables trace exporting
+      # "stdOut": Exports traces to standard output
+      # "stdErr": Exports traces to standard error
+      # "grpc": Exports traces using the OTEL gRPC exporter (recommended)
+      # "http": Exports traces using the OTEL HTTP exporter
+      # "google": Exports traces to Google Cloud Trace
+      Type: "none" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_ENDPOINT
+      # Disable https for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_INSECURE
+      # Interval for batching traces before export
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_TRACE_EXPORTER_GOOGLEPROJECTID
+  Metric:
+    Exporter:
+      # The following exporter types are supported:
+      # "none": Disables metric exporting
+      # "stdOut": Exports metrics to standard output
+      # "stdErr": Exports metrics to standard error
+      # "grpc": Exports metrics using the OTEL gRPC exporter (recommended)
+      # "http": Exports metrics using the OTEL HTTP exporter
+      # "google": Exports metrics to Google Cloud Trace
+      # "prometheus": Exposes metrics via an HTTP endpoint for Prometheus to scrape
+      Type: "none" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_ENDPOINT
+      # Disable https for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_METRIC_EXPORTER_GOOGLEPROJECTID
+  
+  # Structured logging can print logs on standard error and/or export them using OTEL.
+  Log:
+    # Log lines lower than this level are not emitted.
+    Level: "INFO" # ZITADEL_INSTRUMENTATION_LOG_LEVEL
+    # Enable printing structured logs to standard error in the specified format.
+    # The following formats are supported:
+    #   - "disabled": Disables logging to standard error
+    #   - "text": Logs are printed as human-readable text
+    #   - "json": Logs are printed as JSON objects
+    #   - "gcp": JSON formatted logs compatible with Google Cloud Platform logging
+    StdErr: "text" # ZITADEL_INSTRUMENTATION_LOG_STDERR
+    # Adds the source file and line number where the log line was emitted.
+    # Not to be confused with the source of an error.
+    AddSource: true # ZITADEL_INSTRUMENTATION_LOG_ADDSOURCE
+    Exporter:
+      # The following exporter types are supported:
+      #   - "none": Disables log exporting
+      #   - "stdOut": Exports logs to standard output
+      #   - "stdErr": Exports logs to standard error
+      #   - "grpc": Exports logs using the OTEL gRPC exporter (recommended)
+      #   - "http": Exports logs using the OTEL HTTP exporter
+      Type: "none" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_TYPE
+      # Endpoint of the OTEL collector for grpc and http exporters
+      Endpoint: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_ENDPOINT
+      # Disable https for grpc and http exporters
+      Insecure: false # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_INSECURE
+      # Interval at which metrics are exported
+      BatchDuration: 1s # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_BATCHDURATION
+      # Project ID for Google Cloud Trace exporter
+      GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_LOG_EXPORTER_GOOGLEPROJECTID
+  Profile:
+    # The following profiler types are supported:
+    # "none": Disables profiling
+    # "google": Exports profiling data to Google Cloud Profiler
+    Type: "none" # ZITADEL_INSTRUMENTATION_PROFILE_TYPE
+    # Project ID for Google Cloud Profiler
+    GoogleProjectID: "" # ZITADEL_INSTRUMENTATION_PROFILE_GOOGLEPROJECTID
+
 # By using the FirstInstance section, you can overwrite the DefaultInstance configuration for the first instance created by zitadel setup.
 FirstInstance:
   # If set to true zitadel is setup without initial data

cmd/start/config.go

@@ -1,12 +1,16 @@
 package start
 
 import (
+	"context"
+	"errors"
+	"fmt"
 	"time"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/viper"
 	"github.com/zitadel/logging"
 
+	"github.com/zitadel/zitadel/backend/v3/instrumentation"
 	"github.com/zitadel/zitadel/cmd/encryption"
 	"github.com/zitadel/zitadel/cmd/hooks"
 	"github.com/zitadel/zitadel/internal/actions"
@@ -34,12 +38,10 @@ import (
 	"github.com/zitadel/zitadel/internal/query/projection"
 	"github.com/zitadel/zitadel/internal/serviceping"
 	static_config "github.com/zitadel/zitadel/internal/static/config"
-	metrics "github.com/zitadel/zitadel/internal/telemetry/metrics/config"
-	profiler "github.com/zitadel/zitadel/internal/telemetry/profiler/config"
-	tracing "github.com/zitadel/zitadel/internal/telemetry/tracing/config"
 )
 
 type Config struct {
+	Instrumentation     instrumentation.Config
 	Log                 *logging.Config
 	Port                uint16
 	ExternalPort        uint16
@@ -53,9 +55,9 @@ type Config struct {
 	WebAuthNName        string
 	Database            database.Config
 	Caches              *connector.CachesConfig
-	Tracing             tracing.Config
-	Metrics             metrics.Config
-	Profiler            profiler.Config
+	Tracing             *instrumentation.LegacyTraceConfig
+	Metrics             *instrumentation.LegacyMetricConfig
+	Profiler            *instrumentation.LegacyProfileConfig
 	Projections         projection.Config
 	Notifications       handlers.WorkerConfig
 	Executions          execution.WorkerConfig
@@ -93,21 +95,26 @@ type QuotasConfig struct {
 	Execution *logstore.EmitterConfig
 }
 
-func MustNewConfig(v *viper.Viper) *Config {
+func NewConfig(ctx context.Context, v *viper.Viper) (*Config, instrumentation.ShutdownFunc, error) {
 	config, err := readConfig(v)
-	logging.OnError(err).Fatal("unable to read config")
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to read config: %w", err)
+	}
 
+	config.Instrumentation.Trace.SetLegacyConfig(config.Tracing)
+	config.Instrumentation.Metric.SetLegacyConfig(config.Metrics)
+	config.Instrumentation.Profile.SetLegacyConfig(config.Profiler)
+	shutdown, err := instrumentation.Start(ctx, config.Instrumentation)
+	if err != nil {
+		return nil, nil, fmt.Errorf("unable to start instrumentation: %w", err)
+	}
+
+	// Legacy logger
 	err = config.Log.SetLogger()
-	logging.OnError(err).Fatal("unable to set logger")
-
-	err = config.Tracing.NewTracer()
-	logging.OnError(err).Fatal("unable to set tracer")
-
-	err = config.Metrics.NewMeter()
-	logging.OnError(err).Fatal("unable to set meter")
-
-	err = config.Profiler.NewProfiler()
-	logging.OnError(err).Fatal("unable to set profiler")
+	if err != nil {
+		err = errors.Join(err, shutdown(ctx))
+		return nil, nil, fmt.Errorf("unable to set logger: %w", err)
+	}
 
 	id.Configure(config.Machine)
 	if config.Actions != nil {
@@ -115,12 +122,14 @@ func MustNewConfig(v *viper.Viper) *Config {
 	}
 
 	err = config.SystemDefaults.Validate()
-	logging.OnError(err).Fatal("system defaults config invalid")
-
+	if err != nil {
+		err = errors.Join(err, shutdown(ctx))
+		return nil, nil, fmt.Errorf("system defaults config invalid: %w", err)
+	}
 	// Copy the global role permissions mappings to the instance until we allow instance-level configuration over the API.
 	config.DefaultInstance.RolePermissionMappings = config.InternalAuthZ.RolePermissionMappings
 
-	return config
+	return config, shutdown, nil
 }
 
 func readConfig(v *viper.Viper) (*Config, error) {

cmd/start/start.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"crypto/tls"
 	_ "embed"
+	"errors"
 	"fmt"
+	"log/slog"
 	"math"
 	"net/http"
 	"os"
@@ -127,12 +129,25 @@ func New(server chan<- *Server) *cobra.Command {
 		Long: `starts ZITADEL.
 Requirements:
 - postgreSQL`,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			err := cmd_tls.ModeFromFlag(cmd)
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start command failed", "err", err)
+				}
+			}()
+
+			err = cmd_tls.ModeFromFlag(cmd)
 			if err != nil {
 				return err
 			}
-			config := MustNewConfig(viper.GetViper())
+			config, shutdown, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
+
 			masterKey, err := key.MasterKey(cmd)
 			if err != nil {
 				return err

cmd/start/start_from_init.go

@@ -2,10 +2,12 @@ package start
 
 import (
 	"context"
+	"errors"
+	"fmt"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/cmd/initialise"
 	"github.com/zitadel/zitadel/cmd/key"
@@ -24,31 +26,52 @@ Last ZITADEL starts.
 
 Requirements:
 - postgreSQL`,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start-from-init command failed", "err", err)
+				}
+			}()
+
+			err = tls.ModeFromFlag(cmd)
+			if err != nil {
+				return fmt.Errorf("invalid tlsMode: %w", err)
+			}
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
+			if err != nil {
+				return fmt.Errorf("no master key provided: %w", err)
+			}
 
 			initCtx, cancel := context.WithCancel(cmd.Context())
 			initialise.InitAll(initCtx, initialise.MustNewConfig(viper.GetViper()))
 			cancel()
 
 			err = setup.BindInitProjections(cmd)
-			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+			if err != nil {
+				return fmt.Errorf("unable to bind \"init-projections\" flag: %w", err)
+			}
+
+			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			setupConfig := setup.MustNewConfig(viper.GetViper())
 			setupSteps := setup.MustNewSteps(viper.New())
 
 			setupCtx, cancel := context.WithCancel(cmd.Context())
 			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
 			cancel()
 
-			startConfig := MustNewConfig(viper.GetViper())
+			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
 
-			err = startZitadel(cmd.Context(), startConfig, masterKey, server)
-			logging.OnError(err).Fatal("unable to start zitadel")
+			return startZitadel(cmd.Context(), startConfig, masterKey, server)
 		},
 	}
 

cmd/start/start_from_setup.go

@@ -2,10 +2,11 @@ package start
 
 import (
 	"context"
+	"errors"
+	"log/slog"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-	"github.com/zitadel/logging"
 
 	"github.com/zitadel/zitadel/cmd/key"
 	"github.com/zitadel/zitadel/cmd/setup"
@@ -24,27 +25,48 @@ Requirements:
 - database
 - database is initialized
 `,
-		Run: func(cmd *cobra.Command, args []string) {
-			err := tls.ModeFromFlag(cmd)
-			logging.OnError(err).Fatal("invalid tlsMode")
+		RunE: func(cmd *cobra.Command, args []string) (err error) {
+			defer func() {
+				if err != nil {
+					slog.Error("zitadel start-from-setup command failed", "err", err)
+				}
+			}()
+
+			err = tls.ModeFromFlag(cmd)
+			if err != nil {
+				return err
+			}
 
 			masterKey, err := key.MasterKey(cmd)
-			logging.OnError(err).Panic("No master key provided")
+			if err != nil {
+				return err
+			}
 
 			err = setup.BindInitProjections(cmd)
-			logging.OnError(err).Fatal("unable to bind \"init-projections\" flag")
+			if err != nil {
+				return err
+			}
+
+			setupConfig, shutdown, err := setup.NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
+			defer func() {
+				err = errors.Join(err, shutdown(cmd.Context()))
+			}()
 
-			setupConfig := setup.MustNewConfig(viper.GetViper())
 			setupSteps := setup.MustNewSteps(viper.New())
 
 			setupCtx, cancel := context.WithCancel(cmd.Context())
 			setup.Setup(setupCtx, setupConfig, setupSteps, masterKey)
 			cancel()
 
-			startConfig := MustNewConfig(viper.GetViper())
+			startConfig, _, err := NewConfig(cmd.Context(), viper.GetViper())
+			if err != nil {
+				return err
+			}
 
-			err = startZitadel(cmd.Context(), startConfig, masterKey, server)
-			logging.OnError(err).Fatal("unable to start zitadel")
+			return startZitadel(cmd.Context(), startConfig, masterKey, server)
 		},
 	}
 

cmd/zitadel.go

@@ -36,7 +36,8 @@ func New(out io.Writer, in io.Reader, args []string, server chan<- *start.Server
 		RunE: func(cmd *cobra.Command, args []string) error {
 			return errors.New("no additional command provided")
 		},
-		Version: build.Version(),
+		Version:      build.Version(),
+		SilenceUsage: true,
 	}
 
 	viper.AutomaticEnv()