feat(cli): setup (#3267)

* commander

* commander

* selber!

* move to packages

* fix(errors): implement Is interface

* test: command

* test: commands

* add init steps

* setup tenant

* add default step yaml

* possibility to set password

* merge v2 into v2-commander

* fix: rename iam command side to instance

* fix: rename iam command side to instance

* fix: rename iam command side to instance

* fix: rename iam command side to instance

* fix: search query builder can filter events in memory

* fix: filters for add member

* fix(setup): add `ExternalSecure` to config

* chore: name iam to instance

* fix: matching

* remove unsued func

* base url

* base url

* test(command): filter funcs

* test: commands

* fix: rename orgiampolicy to domain policy

* start from init

* commands

* config

* fix indexes and add constraints

* fixes

* fix: merge conflicts

* fix: protos

* fix: md files

* setup

* add deprecated org iam policy again

* typo

* fix search query

* fix filter

* Apply suggestions from code review

* remove custom org from org setup

* add todos for verification

* change apps creation

* simplify package structure

* fix error

* move preparation helper for tests

* fix unique constraints

* fix config mapping in setup

* fix error handling in encryption_keys.go

* fix projection config

* fix query from old views to projection

* fix setup of mgmt api

* set iam project and fix instance projection

* imports

Co-authored-by: Livio Amstutz <livio.a@gmail.com>
Co-authored-by: fabi <fabienne.gerschwiler@gmail.com>

cmd/admin/admin.go

@@ -26,6 +26,7 @@ func New() *cobra.Command {
 		initialise.New(),
 		setup.New(),
 		start.New(),
+		start.NewStartFromInit(),
 		key.New(),
 	)
 

cmd/admin/initialise/config.go

@@ -1,13 +1,29 @@
 package initialise
 
-import "github.com/caos/zitadel/internal/database"
+import (
+	"github.com/caos/logging"
+	"github.com/caos/zitadel/internal/database"
+	"github.com/spf13/viper"
+)
 
 type Config struct {
 	Database  database.Config
 	AdminUser database.User
+	Log       *logging.Config
 }
 
-func adminConfig(config Config) database.Config {
+func MustNewConfig(v *viper.Viper) *Config {
+	config := new(Config)
+	err := v.Unmarshal(config)
+	logging.OnError(err).Fatal("unable to read config")
+
+	err = config.Log.SetLogger()
+	logging.OnError(err).Fatal("unable to set logger")
+
+	return config
+}
+
+func adminConfig(config *Config) database.Config {
 	adminConfig := config.Database
 	adminConfig.Username = config.AdminUser.Username
 	adminConfig.Password = config.AdminUser.Password

cmd/admin/initialise/init.go

@@ -8,9 +8,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	//sql import
-	_ "github.com/lib/pq"
-
 	"github.com/caos/zitadel/internal/database"
 )
 
@@ -28,20 +25,10 @@ The user provided by flags needs priviledge to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			config := Config{}
-			if err := viper.Unmarshal(&config); err != nil {
-				return err
-			}
-			if err := initialise(config,
-				VerifyUser(config.Database.User.Username, config.Database.User.Password),
-				VerifyDatabase(config.Database.Database),
-				VerifyGrant(config.Database.Database, config.Database.User.Username),
-			); err != nil {
-				return err
-			}
+		Run: func(cmd *cobra.Command, args []string) {
+			config := MustNewConfig(viper.GetViper())
 
-			return verifyZitadel(config.Database)
+			InitAll(config)
 		},
 	}
 
@@ -49,7 +36,19 @@ The user provided by flags needs priviledge to
 	return cmd
 }
 
-func initialise(config Config, steps ...func(*sql.DB) error) error {
+func InitAll(config *Config) {
+	err := initialise(config,
+		VerifyUser(config.Database.Username, config.Database.Password),
+		VerifyDatabase(config.Database.Database),
+		VerifyGrant(config.Database.Database, config.Database.Username),
+	)
+	logging.OnError(err).Fatal("unable to initialize the database")
+
+	err = verifyZitadel(config.Database)
+	logging.OnError(err).Fatal("unable to initialize ZITADEL")
+}
+
+func initialise(config *Config, steps ...func(*sql.DB) error) error {
 	logging.Info("initialization started")
 
 	db, err := database.Connect(adminConfig(config))

cmd/admin/initialise/sql/11_unique_constraints_table.sql

@@ -1,5 +1,6 @@
 CREATE TABLE eventstore.unique_constraints (
+    instance_id TEXT,
     unique_type TEXT,
     unique_field TEXT,
-    PRIMARY KEY (unique_type, unique_field)
+    PRIMARY KEY (instance_id, unique_type, unique_field)
 )

cmd/admin/initialise/verify_database.go

@@ -5,6 +5,7 @@ import (
 	_ "embed"
 	"fmt"
 
+	"github.com/caos/logging"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 )
@@ -30,12 +31,11 @@ The user provided by flags needs priviledge to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			config := Config{}
-			if err := viper.Unmarshal(&config); err != nil {
-				return err
-			}
-			return initialise(config, VerifyDatabase(config.Database.Database))
+		Run: func(cmd *cobra.Command, args []string) {
+			config := MustNewConfig(viper.New())
+
+			err := initialise(config, VerifyDatabase(config.Database.Database))
+			logging.OnError(err).Fatal("unable to initialize the database")
 		},
 	}
 }

cmd/admin/initialise/verify_grant.go

@@ -25,12 +25,11 @@ func newGrant() *cobra.Command {
 Prereqesits:
 - cockroachdb
 `,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			config := Config{}
-			if err := viper.Unmarshal(&config); err != nil {
-				return err
-			}
-			return initialise(config, VerifyGrant(config.Database.Database, config.Database.User.Username))
+		Run: func(cmd *cobra.Command, args []string) {
+			config := MustNewConfig(viper.New())
+
+			err := initialise(config, VerifyGrant(config.Database.Database, config.Database.User.Username))
+			logging.OnError(err).Fatal("unable to set grant")
 		},
 	}
 }

cmd/admin/initialise/verify_user.go

@@ -29,12 +29,11 @@ The user provided by flags needs priviledge to
 - see other users and create a new one if the user does not exist
 - grant all rights of the ZITADEL database to the user created if not yet set
 `,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			config := Config{}
-			if err := viper.Unmarshal(&config); err != nil {
-				return err
-			}
-			return initialise(config, VerifyUser(config.Database.User.Username, config.Database.User.Password))
+		Run: func(cmd *cobra.Command, args []string) {
+			config := MustNewConfig(viper.New())
+
+			err := initialise(config, VerifyUser(config.Database.Username, config.Database.Password))
+			logging.OnError(err).Fatal("unable to init user")
 		},
 	}
 }

cmd/admin/initialise/verify_zitadel.go

@@ -94,7 +94,7 @@ func VerifyZitadel(db *sql.DB) error {
 }
 
 func verifyZitadel(config database.Config) error {
-	logging.WithFields("database", config.Database).Info("verify database")
+	logging.WithFields("database", config.Database).Info("verify zitadel")
 	db, err := database.Connect(config)
 	if err != nil {
 		return err

cmd/admin/setup/01_sql/adminapi.sql

@@ -3,27 +3,27 @@ CREATE SCHEMA adminapi;
 CREATE TABLE adminapi.locks (
     locker_id TEXT,
     locked_until TIMESTAMPTZ(3),
-    projection_name TEXT,
+    view_name TEXT,
 
-    PRIMARY KEY (projection_name)
+    PRIMARY KEY (view_name)
 );
 
 CREATE TABLE adminapi.current_sequences (
-    projection_name TEXT,
-    aggregate_type TEXT,
+    view_name TEXT,
     current_sequence BIGINT,
-    timestamp TIMESTAMPTZ,
+    event_timestamp TIMESTAMPTZ,
+    last_successful_spooler_run TIMESTAMPTZ,
 
-    PRIMARY KEY (projection_name, aggregate_type)
+    PRIMARY KEY (view_name)
 );
 
 CREATE TABLE adminapi.failed_events (
-    projection_name TEXT,
+    view_name TEXT,
     failed_sequence BIGINT,
     failure_count SMALLINT,
-    error TEXT,
+    err_msg TEXT,
 
-    PRIMARY KEY (projection_name, failed_sequence)
+    PRIMARY KEY (view_name, failed_sequence)
 );
 
 CREATE TABLE adminapi.styling (

cmd/admin/setup/01_sql/auth.sql

@@ -3,27 +3,27 @@ CREATE SCHEMA auth;
 CREATE TABLE auth.locks (
     locker_id TEXT,
     locked_until TIMESTAMPTZ(3),
-    projection_name TEXT,
+    view_name TEXT,
 
-    PRIMARY KEY (projection_name)
+    PRIMARY KEY (view_name)
 );
 
 CREATE TABLE auth.current_sequences (
-    projection_name TEXT,
-    aggregate_type TEXT,
+    view_name TEXT,
     current_sequence BIGINT,
-    timestamp TIMESTAMPTZ,
+    event_timestamp TIMESTAMPTZ,
+    last_successful_spooler_run TIMESTAMPTZ,
 
-    PRIMARY KEY (projection_name, aggregate_type)
+    PRIMARY KEY (view_name)
 );
 
 CREATE TABLE auth.failed_events (
-    projection_name TEXT,
+    view_name TEXT,
     failed_sequence BIGINT,
     failure_count SMALLINT,
-    error TEXT,
+    err_msg TEXT,
 
-    PRIMARY KEY (projection_name, failed_sequence)
+    PRIMARY KEY (view_name, failed_sequence)
 );
 
 CREATE TABLE auth.users (

cmd/admin/setup/01_sql/authz.sql

@@ -3,27 +3,27 @@ CREATE SCHEMA authz;
 CREATE TABLE authz.locks (
     locker_id TEXT,
     locked_until TIMESTAMPTZ(3),
-    projection_name TEXT,
+    view_name TEXT,
 
-    PRIMARY KEY (projection_name)
+    PRIMARY KEY (view_name)
 );
 
 CREATE TABLE authz.current_sequences (
-    projection_name TEXT,
-    aggregate_type TEXT,
+    view_name TEXT,
     current_sequence BIGINT,
-    timestamp TIMESTAMPTZ,
+    event_timestamp TIMESTAMPTZ,
+    last_successful_spooler_run TIMESTAMPTZ,
 
-    PRIMARY KEY (projection_name, aggregate_type)
+    PRIMARY KEY (view_name)
 );
 
 CREATE TABLE authz.failed_events (
-    projection_name TEXT,
+    view_name TEXT,
     failed_sequence BIGINT,
     failure_count SMALLINT,
-    error TEXT,
+    err_msg TEXT,
 
-    PRIMARY KEY (projection_name, failed_sequence)
+    PRIMARY KEY (view_name, failed_sequence)
 );
 
 CREATE TABLE authz.user_memberships (

cmd/admin/setup/01_sql/notification.sql

@@ -3,27 +3,27 @@ CREATE SCHEMA notification;
 CREATE TABLE notification.locks (
     locker_id TEXT,
     locked_until TIMESTAMPTZ(3),
-    projection_name TEXT,
+    view_name TEXT,
 
-    PRIMARY KEY (projection_name)
+    PRIMARY KEY (view_name)
 );
 
 CREATE TABLE notification.current_sequences (
-    projection_name TEXT,
-    aggregate_type TEXT,
+    view_name TEXT,
     current_sequence BIGINT,
-    timestamp TIMESTAMPTZ,
+    event_timestamp TIMESTAMPTZ,
+    last_successful_spooler_run TIMESTAMPTZ,
 
-    PRIMARY KEY (projection_name, aggregate_type)
+    PRIMARY KEY (view_name)
 );
 
 CREATE TABLE notification.failed_events (
-    projection_name TEXT,
+    view_name TEXT,
     failed_sequence BIGINT,
     failure_count SMALLINT,
-    error TEXT,
+    err_msg TEXT,
 
-    PRIMARY KEY (projection_name, failed_sequence)
+    PRIMARY KEY (view_name, failed_sequence)
 );
 
 CREATE TABLE notification.notify_users (

cmd/admin/setup/02.go

@@ -0,0 +1,22 @@
+package setup
+
+import (
+	"context"
+
+	command "github.com/caos/zitadel/internal/command/v2"
+)
+
+type DefaultInstance struct {
+	cmd           *command.Command
+	InstanceSetup command.InstanceSetup
+}
+
+func (mig *DefaultInstance) Execute(ctx context.Context) error {
+	_, err := mig.cmd.SetUpInstance(ctx, &mig.InstanceSetup)
+
+	return err
+}
+
+func (mig *DefaultInstance) String() string {
+	return "02_default_instance"
+}

cmd/admin/setup/config.go

@@ -1,13 +1,58 @@
 package setup
 
 import (
+	"bytes"
+
+	"github.com/caos/logging"
+	"github.com/mitchellh/mapstructure"
+	"github.com/spf13/viper"
+
+	"github.com/caos/zitadel/internal/api/authz"
+	"github.com/caos/zitadel/internal/config/hook"
+	"github.com/caos/zitadel/internal/config/systemdefaults"
 	"github.com/caos/zitadel/internal/database"
 )
 
 type Config struct {
-	Database database.Config
+	Database       database.Config
+	SystemDefaults systemdefaults.SystemDefaults
+	InternalAuthZ  authz.Config
+	ExternalPort   uint16
+	ExternalDomain string
+	ExternalSecure bool
+	Log            *logging.Config
+}
+
+func MustNewConfig(v *viper.Viper) *Config {
+	config := new(Config)
+	err := v.Unmarshal(config)
+	logging.OnError(err).Fatal("unable to read config")
+
+	err = config.Log.SetLogger()
+	logging.OnError(err).Fatal("unable to set logger")
+
+	return config
 }
 
 type Steps struct {
 	S1ProjectionTable *ProjectionTable
+	S2DefaultInstance *DefaultInstance
+}
+
+func MustNewSteps(v *viper.Viper) *Steps {
+	v.SetConfigType("yaml")
+	err := v.ReadConfig(bytes.NewBuffer(defaultSteps))
+	logging.OnError(err).Fatal("unable to read setup steps")
+
+	steps := new(Steps)
+	err = v.Unmarshal(steps,
+		viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
+			hook.Base64ToBytesHookFunc(),
+			hook.TagToLanguageHookFunc(),
+			mapstructure.StringToTimeDurationHookFunc(),
+			mapstructure.StringToSliceHookFunc(","),
+		)),
+	)
+	logging.OnError(err).Fatal("unable to read steps")
+	return steps
 }

cmd/admin/setup/setup.go

@@ -1,7 +1,6 @@
 package setup
 
 import (
-	"bytes"
 	"context"
 	_ "embed"
 
@@ -9,6 +8,8 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	http_util "github.com/caos/zitadel/internal/api/http"
+	command "github.com/caos/zitadel/internal/command/v2"
 	"github.com/caos/zitadel/internal/database"
 	"github.com/caos/zitadel/internal/eventstore"
 	"github.com/caos/zitadel/internal/migration"
@@ -27,32 +28,30 @@ func New() *cobra.Command {
 Requirements:
 - cockroachdb`,
 		Run: func(cmd *cobra.Command, args []string) {
-			config := new(Config)
-			err := viper.Unmarshal(config)
-			logging.OnError(err).Fatal("unable to read config")
+			config := MustNewConfig(viper.GetViper())
+			steps := MustNewSteps(viper.New())
 
-			v := viper.New()
-			v.SetConfigType("yaml")
-			err = v.ReadConfig(bytes.NewBuffer(defaultSteps))
-			logging.OnError(err).Fatal("unable to read setup steps")
-
-			steps := new(Steps)
-			err = v.Unmarshal(steps)
-			logging.OnError(err).Fatal("unable to read steps")
-
-			setup(config, steps)
+			Setup(config, steps)
 		},
 	}
 }
 
-func setup(config *Config, steps *Steps) {
+func Setup(config *Config, steps *Steps) {
 	dbClient, err := database.Connect(config.Database)
 	logging.OnError(err).Fatal("unable to connect to database")
 
 	eventstoreClient, err := eventstore.Start(dbClient)
 	logging.OnError(err).Fatal("unable to start eventstore")
+	migration.RegisterMappers(eventstoreClient)
 
+	cmd := command.New(eventstoreClient, "localhost", config.SystemDefaults)
+
+	steps.S2DefaultInstance.cmd = cmd
 	steps.S1ProjectionTable = &ProjectionTable{dbClient: dbClient}
+	steps.S2DefaultInstance.InstanceSetup.Zitadel.IsDevMode = !config.ExternalSecure
+	steps.S2DefaultInstance.InstanceSetup.Zitadel.BaseURL = http_util.BuildHTTP(config.ExternalDomain, config.ExternalPort, config.ExternalSecure)
 
-	migration.Migrate(context.Background(), eventstoreClient, steps.S1ProjectionTable)
+	ctx := context.Background()
+	migration.Migrate(ctx, eventstoreClient, steps.S1ProjectionTable)
+	migration.Migrate(ctx, eventstoreClient, steps.S2DefaultInstance)
 }

cmd/admin/setup/steps.yaml

@@ -0,0 +1,127 @@
+S2DefaultInstance:
+  InstanceSetup:
+    Org:
+      Name: ZITADEL
+      Human:
+        UserName: zitadel-admin
+        FirstName: ZITADEL
+        LastName: Admin
+        NickName:
+        DisplayName:
+        Email: admin@zitadel.ch
+        PreferredLanguage:
+        Gender:
+        Phone:
+        Password: Password1!
+    PasswordComplexityPolicy: 
+      MinLength: 8
+      HasLowercase: true 
+      HasUppercase: true
+      HasNumber: true
+      HasSymbol: true
+    PasswordAgePolicy:
+      ExpireWarnDays: 0
+      MaxAgeDays: 0
+    OrgIAMPolicy:
+      UserLoginMustBeDomain: true
+    LoginPolicy:
+      AllowUsernamePassword: true
+      AllowRegister: true
+      AllowExternalIDP: true
+      ForceMFA: false
+      HidePasswordReset: false
+      PasswordlessType: 1 #1: allowed 0: not allowed
+      PasswordCheckLifetime: 240h #10d
+      ExternalLoginCheckLifetime: 240h #10d
+      MfaInitSkipLifetime: 720h #30d
+      SecondFactorCheckLifetime: 18h
+      MultiFactorCheckLifetime: 12h
+    PrivacyPolicy:
+      TOSLink: https://docs.zitadel.ch/docs/legal/terms-of-service
+      PrivacyLink: https://docs.zitadel.ch/docs/legal/privacy-policy
+      HelpLink: ''
+    LockoutPolicy:
+      MaxAttempts: 0
+      ShouldShowLockoutFailure: true
+    EmailTemplate: CjwhZG9jdHlwZSBodG1sPgo8aHRtbCB4bWxucz0iaHR0cDovL3d3dy53My5vcmcvMTk5OS94aHRtbCIgeG1sbnM6dj0idXJuOnNjaGVtYXMtbWljcm9zb2Z0LWNvbTp2bWwiIHhtbG5zOm89InVybjpzY2hlbWFzLW1pY3Jvc29mdC1jb206b2ZmaWNlOm9mZmljZSI+CjxoZWFkPgogIDx0aXRsZT4KCiAgPC90aXRsZT4KICA8IS0tW2lmICFtc29dPjwhLS0+CiAgPG1ldGEgaHR0cC1lcXVpdj0iWC1VQS1Db21wYXRpYmxlIiBjb250ZW50PSJJRT1lZGdlIj4KICA8IS0tPCFbZW5kaWZdLS0+CiAgPG1ldGEgaHR0cC1lcXVpdj0iQ29udGVudC1UeXBlIiBjb250ZW50PSJ0ZXh0L2h0bWw7IGNoYXJzZXQ9VVRGLTgiPgogIDxtZXRhIG5hbWU9InZpZXdwb3J0IiBjb250ZW50PSJ3aWR0aD1kZXZpY2Utd2lkdGgsIGluaXRpYWwtc2NhbGU9MSI+CiAgPHN0eWxlIHR5cGU9InRleHQvY3NzIj4KICAgICNvdXRsb29rIGEgeyBwYWRkaW5nOjA7IH0KICAgIGJvZHkgeyBtYXJnaW46MDtwYWRkaW5nOjA7LXdlYmtpdC10ZXh0LXNpemUtYWRqdXN0OjEwMCU7LW1zLXRleHQtc2l6ZS1hZGp1c3Q6MTAwJTsgfQogICAgdGFibGUsIHRkIHsgYm9yZGVyLWNvbGxhcHNlOmNvbGxhcHNlO21zby10YWJsZS1sc3BhY2U6MHB0O21zby10YWJsZS1yc3BhY2U6MHB0OyB9CiAgICBpbWcgeyBib3JkZXI6MDtoZWlnaHQ6YXV0bztsaW5lLWhlaWdodDoxMDAlOyBvdXRsaW5lOm5vbmU7dGV4dC1kZWNvcmF0aW9uOm5vbmU7LW1zLWludGVycG9sYXRpb24tbW9kZTpiaWN1YmljOyB9CiAgICBwIHsgZGlzcGxheTpibG9jazttYXJnaW46MTNweCAwOyB9CiAgPC9zdHlsZT4KICA8IS0tW2lmIG1zb10+CiAgPHhtbD4KICAgIDxvOk9mZmljZURvY3VtZW50U2V0dGluZ3M+CiAgICAgIDxvOkFsbG93UE5HLz4KICAgICAgPG86UGl4ZWxzUGVySW5jaD45NjwvbzpQaXhlbHNQZXJJbmNoPgogICAgPC9vOk9mZmljZURvY3VtZW50U2V0dGluZ3M+CiAgPC94bWw+CiAgPCFbZW5kaWZdLS0+CiAgPCEtLVtpZiBsdGUgbXNvIDExXT4KICA8c3R5bGUgdHlwZT0idGV4dC9jc3MiPgogICAgLm1qLW91dGxvb2stZ3JvdXAtZml4IHsgd2lkdGg6MTAwJSAhaW1wb3J0YW50OyB9CiAgPC9zdHlsZT4KICA8IVtlbmRpZl0tLT4KCiAgPCEtLVtpZiAhbXNvXT48IS0tPgogIDxsaW5rIGhyZWY9Imh0dHBzOi8vZm9udHMuZ29vZ2xlYXBpcy5jb20vY3NzP2ZhbWlseT1VYnVudHU6MzAwLDQwMCw1MDAsNzAwIiByZWw9InN0eWxlc2hlZXQiIHR5cGU9InRleHQvY3NzIj4KICA8c3R5bGUgdHlwZT0idGV4dC9jc3MiPgogICAgQGltcG9ydCB1cmwoaHR0cHM6Ly9mb250cy5nb29nbGVhcGlzLmNvbS9jc3M/ZmFtaWx5PVVidW50dTozMDAsNDAwLDUwMCw3MDApOwogIDwvc3R5bGU+CiAgPCEtLTwhW2VuZGlmXS0tPgoKCgogIDxzdHlsZSB0eXBlPSJ0ZXh0L2NzcyI+CiAgICBAbWVkaWEgb25seSBzY3JlZW4gYW5kIChtaW4td2lkdGg6NDgwcHgpIHsKICAgICAgLm1qLWNvbHVtbi1wZXItMTAwIHsgd2lkdGg6MTAwJSAhaW1wb3J0YW50OyBtYXgtd2lkdGg6IDEwMCU7IH0KICAgICAgLm1qLWNvbHVtbi1wZXItNjAgeyB3aWR0aDo2MCUgIWltcG9ydGFudDsgbWF4LXdpZHRoOiA2MCU7IH0KICAgIH0KICA8L3N0eWxlPgoKCiAgPHN0eWxlIHR5cGU9InRleHQvY3NzIj4KCgoKICAgIEBtZWRpYSBvbmx5IHNjcmVlbiBhbmQgKG1heC13aWR0aDo0ODBweCkgewogICAgICB0YWJsZS5tai1mdWxsLXdpZHRoLW1vYmlsZSB7IHdpZHRoOiAxMDAlICFpbXBvcnRhbnQ7IH0KICAgICAgdGQubWotZnVsbC13aWR0aC1tb2JpbGUgeyB3aWR0aDogYXV0byAhaW1wb3J0YW50OyB9CiAgICB9CgogIDwvc3R5bGU+CiAgPHN0eWxlIHR5cGU9InRleHQvY3NzIj4uc2hhZG93IGEgewogICAgYm94LXNoYWRvdzogMHB4IDNweCAxcHggLTJweCByZ2JhKDAsIDAsIDAsIDAuMiksIDBweCAycHggMnB4IDBweCByZ2JhKDAsIDAsIDAsIDAuMTQpLCAwcHggMXB4IDVweCAwcHggcmdiYSgwLCAwLCAwLCAwLjEyKTsKICB9PC9zdHlsZT4KCiAge3tpZiAuRm9udFVSTH19CiAgPHN0eWxlPgogICAgQGZvbnQtZmFjZSB7CiAgICAgIGZvbnQtZmFtaWx5OiAne3suRm9udEZhbWlseX19JzsKICAgICAgZm9udC1zdHlsZTogbm9ybWFsOwogICAgICBmb250LWRpc3BsYXk6IHN3YXA7CiAgICAgIHNyYzogdXJsKHt7LkZvbnRVUkx9fSk7CiAgICB9CiAgPC9zdHlsZT4KICB7e2VuZH19Cgo8L2hlYWQ+Cjxib2R5IHN0eWxlPSJ3b3JkLXNwYWNpbmc6bm9ybWFsOyI+CgoKPGRpdgogICAgICAgIHN0eWxlPSIiCj4KCiAgPHRhYmxlCiAgICAgICAgICBhbGlnbj0iY2VudGVyIiBib3JkZXI9IjAiIGNlbGxwYWRkaW5nPSIwIiBjZWxsc3BhY2luZz0iMCIgcm9sZT0icHJlc2VudGF0aW9uIiBzdHlsZT0iYmFja2dyb3VuZDp7ey5CYWNrZ3JvdW5kQ29sb3J9fTtiYWNrZ3JvdW5kLWNvbG9yOnt7LkJhY2tncm91bmRDb2xvcn19O3dpZHRoOjEwMCU7Ym9yZGVyLXJhZGl1czoxNnB4OyIKICA+CiAgICA8dGJvZHk+CiAgICA8dHI+CiAgICAgIDx0ZD4KCgogICAgICAgIDwhLS1baWYgbXNvIHwgSUVdPjx0YWJsZSBhbGlnbj0iY2VudGVyIiBib3JkZXI9IjAiIGNlbGxwYWRkaW5nPSIwIiBjZWxsc3BhY2luZz0iMCIgY2xhc3M9IiIgc3R5bGU9IndpZHRoOjgwMHB4OyIgd2lkdGg9IjgwMCIgPjx0cj48dGQgc3R5bGU9ImxpbmUtaGVpZ2h0OjBweDtmb250LXNpemU6MHB4O21zby1saW5lLWhlaWdodC1ydWxlOmV4YWN0bHk7Ij48IVtlbmRpZl0tLT4KCgogICAgICAgIDxkaXYgIHN0eWxlPSJtYXJnaW46MHB4IGF1dG87Ym9yZGVyLXJhZGl1czoxNnB4O21heC13aWR0aDo4MDBweDsiPgoKICAgICAgICAgIDx0YWJsZQogICAgICAgICAgICAgICAgICBhbGlnbj0iY2VudGVyIiBib3JkZXI9IjAiIGNlbGxwYWRkaW5nPSIwIiBjZWxsc3BhY2luZz0iMCIgcm9sZT0icHJlc2VudGF0aW9uIiBzdHlsZT0id2lkdGg6MTAwJTtib3JkZXItcmFkaXVzOjE2cHg7IgogICAgICAgICAgPgogICAgICAgICAgICA8dGJvZHk+CiAgICAgICAgICAgIDx0cj4KICAgICAgICAgICAgICA8dGQKICAgICAgICAgICAgICAgICAgICAgIHN0eWxlPSJkaXJlY3Rpb246bHRyO2ZvbnQtc2l6ZTowcHg7cGFkZGluZzoyMHB4IDA7cGFkZGluZy1sZWZ0OjA7dGV4dC1hbGlnbjpjZW50ZXI7IgogICAgICAgICAgICAgID4KICAgICAgICAgICAgICAgIDwhLS1baWYgbXNvIHwgSUVdPjx0YWJsZSByb2xlPSJwcmVzZW50YXRpb24iIGJvcmRlcj0iMCIgY2VsbHBhZGRpbmc9IjAiIGNlbGxzcGFjaW5nPSIwIj48dHI+PHRkIGNsYXNzPSIiIHdpZHRoPSI4MDBweCIgPjwhW2VuZGlmXS0tPgoKICAgICAgICAgICAgICAgIDx0YWJsZQogICAgICAgICAgICAgICAgICAgICAgICBhbGlnbj0iY2VudGVyIiBib3JkZXI9IjAiIGNlbGxwYWRkaW5nPSIwIiBjZWxsc3BhY2luZz0iMCIgcm9sZT0icHJlc2VudGF0aW9uIiBzdHlsZT0id2lkdGg6MTAwJTsiCiAgICAgICAgICAgICAgICA+CiAgICAgICAgICAgICAgICAgIDx0Ym9keT4KICAgICAgICAgICAgICAgICAgPHRyPgogICAgICAgICAgICAgICAgICAgIDx0ZD4KCgogICAgICAgICAgICAgICAgICAgICAgPCEtLVtpZiBtc28gfCBJRV0+PHRhYmxlIGFsaWduPSJjZW50ZXIiIGJvcmRlcj0iMCIgY2VsbHBhZGRpbmc9IjAiIGNlbGxzcGFjaW5nPSIwIiBjbGFzcz0iIiBzdHlsZT0id2lkdGg6ODAwcHg7IiB3aWR0aD0iODAwIiA+PHRyPjx0ZCBzdHlsZT0ibGluZS1oZWlnaHQ6MHB4O2ZvbnQtLine truncated
+    MessageTexts:
+      - MessageTextType: InitCode
+        Language: de
+        Title: Zitadel - User initialisieren
+        PreHeader: User initialisieren
+        Subject: User initialisieren
+        Greeting: Hallo {{.FirstName}} {{.LastName}},
+        Text: Dieser Benutzer wurde soeben im Zitadel erstellt. Mit dem Benutzernamen <br><strong>{{.PreferredLoginName}}</strong><br> kannst du dich anmelden. Nutze den untenstehenden Button, um die Initialisierung abzuschliessen <br>(Code <strong>{{.Code}}</strong>).<br> Falls du dieses Mail nicht angefordert hast, kannst du es einfach ignorieren.
+        ButtonText: Initialisierung abschliessen
+      - MessageTextType: PasswordReset
+        Language: de
+        Title: Zitadel - Passwort zurücksetzen
+        PreHeader: Passwort zurücksetzen
+        Subject: Passwort zurücksetzen
+        Greeting: Hallo {{.FirstName}} {{.LastName}},
+        Text: Wir haben eine Anfrage für das Zurücksetzen deines Passwortes bekommen. Du kannst den untenstehenden Button verwenden, um dein Passwort zurückzusetzen <br>(Code <strong>{{.Code}}</strong>).<br> Falls du dieses Mail nicht angefordert hast, kannst du es ignorieren.
+        ButtonText: Passwort zurücksetzen
+      - MessageTextType: VerifyEmail
+        Language: de
+        Title: Zitadel - Email verifizieren
+        PreHeader: Email verifizieren
+        Subject: Email verifizieren
+        Greeting: Hallo {{.FirstName}} {{.LastName}},
+        Text: Eine neue E-Mail Adresse wurde hinzugefügt. Bitte verwende den untenstehenden Button um diese zu verifizieren <br>(Code <strong>{{.Code}}</strong>).<br> Falls du deine E-Mail Adresse nicht selber hinzugefügt hast, kannst du dieses E-Mail ignorieren.
+        ButtonText: Email verifizieren
+      - MessageTextType: VerifyPhone
+        Language: de
+        Title: Zitadel - Telefonnummer verifizieren
+        PreHeader: Telefonnummer verifizieren
+        Subject: Telefonnummer verifizieren
+        Greeting: Hallo {{.FirstName}} {{.LastName}},
+        Text: Eine Telefonnummer wurde hinzugefügt. Bitte verifiziere diese in dem du folgenden Code eingibst (Code {{.Code}})
+        ButtonText: Telefon verifizieren
+      - MessageTextType: DomainClaimed
+        Language: de
+        Title: Zitadel - Domain wurde beansprucht
+        PreHeader: Email / Username ändern
+        Subject: Domain wurde beansprucht
+        Greeting: Hallo {{.FirstName}} {{.LastName}},
+        Text: Die Domain {{.Domain}} wurde von einer Organisation beansprucht. Dein derzeitiger User {{.Username}} ist nicht Teil dieser Organisation. Daher musst du beim nächsten Login eine neue Email hinterlegen. Für diesen Login haben wir dir einen temporären Usernamen ({{.TempUsername}}) erstellt.
+        ButtonText: Login
+      - MessageTextType: InitCode
+        Language: en
+        Title: Zitadel - Initialize User
+        PreHeader: Initialize User
+        Subject: Initialize User
+        Greeting: Hello {{.FirstName}} {{.LastName}},
+        Text: This user was created in Zitadel. Use the username {{.PreferredLoginName}} to login. Please click the button below to finish the initialization process. (Code {{.Code}}) If you didn't ask for this mail, please ignore it.
+        ButtonText: Finish initialization
+      - MessageTextType: PasswordReset
+        Language: en
+        Title: Zitadel - Reset password
+        PreHeader: Reset password
+        Subject: Reset password
+        Greeting: Hello {{.FirstName}} {{.LastName}},
+        Text: We received a password reset request. Please use the button below to reset your password. (Code {{.Code}}) If you didn't ask for this mail, please ignore it.
+        ButtonText: Reset password
+      - MessageTextType: VerifyEmail
+        Language: en
+        Title: Zitadel - Verify email
+        PreHeader: Verify email
+        Subject: Verify email
+        Greeting: Hello {{.FirstName}} {{.LastName}},
+        Text: A new email has been added. Please use the button below to verify your mail. (Code {{.Code}}) If you din't add a new email, please ignore this email.
+        ButtonText: Verify email
+      - MessageTextType: VerifyPhone
+        Language: en
+        Title: Zitadel - Verify phone
+        PreHeader: Verify phone
+        Subject: Verify phone
+        Greeting: Hello {{.FirstName}} {{.LastName}},
+        Text: A new phonenumber has been added. Please use the following code to verify it {{.Code}}.
+        ButtonText: Verify phone
+      - MessageTextType: DomainClaimed
+        Language: en
+        Title: Zitadel - Domain has been claimed
+        PreHeader: Change email / username
+        Subject: Domain has been claimed
+        Greeting: Hello {{.FirstName}} {{.LastName}},
+        Text: The domain {{.Domain}} has been claimed by an organisation. Your current user {{.UserName}} is not part of this organisation. Therefore you'll have to change your email when you login. We have created a temporary username ({{.TempUsername}}) for this login.
+        ButtonText: Login

cmd/admin/start/config.go

@@ -0,0 +1,66 @@
+package start
+
+import (
+	"github.com/caos/logging"
+	admin_es "github.com/caos/zitadel/internal/admin/repository/eventsourcing"
+	internal_authz "github.com/caos/zitadel/internal/api/authz"
+	"github.com/caos/zitadel/internal/api/http/middleware"
+	"github.com/caos/zitadel/internal/api/oidc"
+	"github.com/caos/zitadel/internal/api/ui/console"
+	"github.com/caos/zitadel/internal/api/ui/login"
+	auth_es "github.com/caos/zitadel/internal/auth/repository/eventsourcing"
+	"github.com/caos/zitadel/internal/authz"
+	"github.com/caos/zitadel/internal/config/systemdefaults"
+	"github.com/caos/zitadel/internal/crypto"
+	"github.com/caos/zitadel/internal/database"
+	"github.com/caos/zitadel/internal/notification"
+	"github.com/caos/zitadel/internal/query/projection"
+	static_config "github.com/caos/zitadel/internal/static/config"
+	"github.com/spf13/viper"
+)
+
+type Config struct {
+	Log             *logging.Config
+	Port            uint16
+	ExternalPort    uint16
+	ExternalDomain  string
+	ExternalSecure  bool
+	Database        database.Config
+	Projections     projection.Config
+	AuthZ           authz.Config
+	Auth            auth_es.Config
+	Admin           admin_es.Config
+	UserAgentCookie *middleware.UserAgentCookieConfig
+	OIDC            oidc.Config
+	Login           login.Config
+	Console         console.Config
+	Notification    notification.Config
+	AssetStorage    static_config.AssetStorageConfig
+	InternalAuthZ   internal_authz.Config
+	SystemDefaults  systemdefaults.SystemDefaults
+	EncryptionKeys  *encryptionKeyConfig
+}
+
+func MustNewConfig(v *viper.Viper) *Config {
+	config := new(Config)
+
+	err := v.Unmarshal(config)
+	logging.OnError(err).Fatal("unable to read config")
+
+	err = config.Log.SetLogger()
+	logging.OnError(err).Fatal("unable to set logger")
+
+	return config
+}
+
+type encryptionKeyConfig struct {
+	DomainVerification   *crypto.KeyConfig
+	IDPConfig            *crypto.KeyConfig
+	OIDC                 *crypto.KeyConfig
+	OTP                  *crypto.KeyConfig
+	SMS                  *crypto.KeyConfig
+	SMTP                 *crypto.KeyConfig
+	User                 *crypto.KeyConfig
+	CSRFCookieKeyID      string
+	UserAgentCookieKeyID string
+}

cmd/admin/start/encryption_keys.go

@@ -35,7 +35,7 @@ type encryptionKeys struct {
 func ensureEncryptionKeys(keyConfig *encryptionKeyConfig, keyStorage crypto.KeyStorage) (*encryptionKeys, error) {
 	keys, err := keyStorage.ReadKeys()
 	if err != nil {
-		return nil, nil
+		return nil, err
 	}
 	if len(keys) == 0 {
 		if err := createDefaultKeys(keyStorage); err != nil {

cmd/admin/start/flags.go

@@ -0,0 +1,31 @@
+package start
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func startFlags(cmd *cobra.Command) {
+	bindUint16Flag(cmd, "port", "port to run ZITADEL on")
+	bindStringFlag(cmd, "externalDomain", "domain ZITADEL will be exposed on")
+	bindStringFlag(cmd, "externalPort", "port ZITADEL will be exposed on")
+	bindBoolFlag(cmd, "externalSecure", "if ZITADEL will be served on HTTPS")
+
+	cmd.PersistentFlags().String(flagMasterKey, "", "masterkey for en/decryption keys")
+
+}
+
+func bindStringFlag(cmd *cobra.Command, name, description string) {
+	cmd.PersistentFlags().String(name, viper.GetString(name), description)
+	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
+}
+
+func bindUint16Flag(cmd *cobra.Command, name, description string) {
+	cmd.PersistentFlags().Uint16(name, uint16(viper.GetUint(name)), description)
+	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
+}
+
+func bindBoolFlag(cmd *cobra.Command, name, description string) {
+	cmd.PersistentFlags().Bool(name, viper.GetBool(name), description)
+	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
+}

cmd/admin/start/start.go

@@ -15,7 +15,6 @@ import (
 
 	"github.com/caos/logging"
 	"github.com/gorilla/mux"
-	"github.com/mitchellh/mapstructure"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"golang.org/x/net/http2"
@@ -37,17 +36,13 @@ import (
 	"github.com/caos/zitadel/internal/authz"
 	authz_repo "github.com/caos/zitadel/internal/authz/repository"
 	"github.com/caos/zitadel/internal/command"
-	"github.com/caos/zitadel/internal/config/systemdefaults"
-	"github.com/caos/zitadel/internal/crypto"
 	cryptoDB "github.com/caos/zitadel/internal/crypto/database"
 	"github.com/caos/zitadel/internal/database"
 	"github.com/caos/zitadel/internal/eventstore"
 	"github.com/caos/zitadel/internal/id"
 	"github.com/caos/zitadel/internal/notification"
 	"github.com/caos/zitadel/internal/query"
-	"github.com/caos/zitadel/internal/query/projection"
 	"github.com/caos/zitadel/internal/static"
-	static_config "github.com/caos/zitadel/internal/static/config"
 	"github.com/caos/zitadel/internal/webauthn"
 	"github.com/caos/zitadel/openapi"
 )
@@ -64,82 +59,19 @@ func New() *cobra.Command {
 Requirements:
 - cockroachdb`,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			config := new(startConfig)
-			err := viper.Unmarshal(config, viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
-				mapstructure.StringToTimeDurationHookFunc(),
-				mapstructure.StringToSliceHookFunc(":"),
-			)))
-			if err != nil {
-				return err
-			}
-			err = config.Log.SetLogger()
-			if err != nil {
-				return err
-			}
-			masterKey, _ := cmd.Flags().GetString("masterkey")
+			config := MustNewConfig(viper.GetViper())
+			masterKey, _ := cmd.Flags().GetString(flagMasterKey)
+
 			return startZitadel(config, masterKey)
 		},
 	}
-	bindUint16Flag(start, "port", "port to run ZITADEL on")
-	bindStringFlag(start, "externalDomain", "domain ZITADEL will be exposed on")
-	bindStringFlag(start, "externalPort", "port ZITADEL will be exposed on")
-	bindBoolFlag(start, "externalSecure", "if ZITADEL will be served on HTTPS")
 
-	start.PersistentFlags().String(flagMasterKey, "", "masterkey for en/decryption keys")
+	startFlags(start)
 
 	return start
 }
 
-func bindStringFlag(cmd *cobra.Command, name, description string) {
-	cmd.PersistentFlags().String(name, viper.GetString(name), description)
-	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
-}
-
-func bindUint16Flag(cmd *cobra.Command, name, description string) {
-	cmd.PersistentFlags().Uint16(name, uint16(viper.GetUint(name)), description)
-	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
-}
-
-func bindBoolFlag(cmd *cobra.Command, name, description string) {
-	cmd.PersistentFlags().Bool(name, viper.GetBool(name), description)
-	viper.BindPFlag(name, cmd.PersistentFlags().Lookup(name))
-}
-
-type startConfig struct {
-	Log             *logging.Config
-	Port            uint16
-	ExternalPort    uint16
-	ExternalDomain  string
-	ExternalSecure  bool
-	Database        database.Config
-	Projections     projection.Config
-	AuthZ           authz.Config
-	Auth            auth_es.Config
-	Admin           admin_es.Config
-	UserAgentCookie *middleware.UserAgentCookieConfig
-	OIDC            oidc.Config
-	Login           login.Config
-	Console         console.Config
-	Notification    notification.Config
-	AssetStorage    static_config.AssetStorageConfig
-	InternalAuthZ   internal_authz.Config
-	SystemDefaults  systemdefaults.SystemDefaults
-	EncryptionKeys  *encryptionKeyConfig
-}
-
-type encryptionKeyConfig struct {
-	DomainVerification   *crypto.KeyConfig
-	IDPConfig            *crypto.KeyConfig
-	OIDC                 *crypto.KeyConfig
-	OTP                  *crypto.KeyConfig
-	SMS                  *crypto.KeyConfig
-	SMTP                 *crypto.KeyConfig
-	User                 *crypto.KeyConfig
-	CSRFCookieKeyID      string
-	UserAgentCookieKeyID string
-}
-
-func startZitadel(config *startConfig, masterKey string) error {
+func startZitadel(config *Config, masterKey string) error {
 	ctx := context.Background()
 	keyChan := make(chan interface{})
 
@@ -197,7 +129,7 @@ func startZitadel(config *startConfig, masterKey string) error {
 	return listen(ctx, router, config.Port)
 }
 
-func startAPIs(ctx context.Context, router *mux.Router, commands *command.Commands, queries *query.Queries, eventstore *eventstore.Eventstore, dbClient *sql.DB, keyChan chan interface{}, config *startConfig, store static.Storage, authZRepo authz_repo.Repository, keys *encryptionKeys) error {
+func startAPIs(ctx context.Context, router *mux.Router, commands *command.Commands, queries *query.Queries, eventstore *eventstore.Eventstore, dbClient *sql.DB, keyChan chan interface{}, config *Config, store static.Storage, authZRepo authz_repo.Repository, keys *encryptionKeys) error {
 	repo := struct {
 		authz_repo.Repository
 		*query.Queries

cmd/admin/start/start_from_init.go

@@ -0,0 +1,40 @@
+package start
+
+import (
+	"github.com/caos/logging"
+	"github.com/caos/zitadel/cmd/admin/initialise"
+	"github.com/caos/zitadel/cmd/admin/setup"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func NewStartFromInit() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "start-from-init",
+		Short: "cold starts zitadel",
+		Long: `cold starts ZITADEL.
+First the minimum requirements to start ZITADEL are set up.
+Second the initial events are created.
+Last ZITADEL starts.
+
+Requirements:
+- cockroachdb`,
+		Run: func(cmd *cobra.Command, args []string) {
+			initialise.InitAll(initialise.MustNewConfig(viper.GetViper()))
+
+			setupConfig := setup.MustNewConfig(viper.GetViper())
+			setupSteps := setup.MustNewSteps(viper.New())
+			setup.Setup(setupConfig, setupSteps)
+
+			startConfig := MustNewConfig(viper.GetViper())
+			startMasterKey, _ := cmd.Flags().GetString(flagMasterKey)
+
+			err := startZitadel(startConfig, startMasterKey)
+			logging.OnError(err).Fatal("unable to start zitadel")
+		},
+	}
+
+	startFlags(cmd)
+
+	return cmd
+}