From 25c23cf91dd430a3be4a0e484140397acf43cc79 Mon Sep 17 00:00:00 2001 From: Silvan <27845747+adlerhurst@users.noreply.github.com> Date: Wed, 23 Sep 2026 16:20:50 +0200 Subject: [PATCH] fix: setup step 79 batch unique constraint owner backfill (#12803) - Setup 79 rewrites unique-constraint owners in one statement per file. On a large table that statement holds row locks for the whole update, and adding `LIMIT` does not help: the username statement bitmap-scans every empty `owners` value and then index-scans every user in the instance for each constraint. - The backfill joined projection tables before those tables were created and before event replay filled them. - The page cursor is a `text[]`. `database.TextArray.Value` joined elements with commas and never quoted them, so a project role key such as `a,b` became two array elements. The next page repeated the same row and the migration stopped. - Each setup 79 statement pages its projection by primary key and updates `eventstore.unique_constraints` with an equality lookup on `(instance_id, unique_type, unique_field)`. - The original field and `lower(...)` are separate `UNION` arms, so the lookup stays a primary-key probe. - Setup repeats a file until the projection page is empty. A page that matches no constraint still advances the cursor. - `BatchSize` is a `uint16`, configured like `ForceFinalize` (`cmd/setup/steps.yaml`, `ZITADEL_BACKFILLUNIQUECONSTRAINTOWNERS_BATCHSIZE`) and is not stored on lastRun. The default in `steps.yaml` is 5000. - Each page logs the file, the page index, and the number of rows updated. - The backfill runs after projection prefill. When `InitProjections` is disabled or setup is for mirror, it still runs against the projection rows already stored. - `TextArray.Value` quotes every element and backslash-escapes `"` and `\`. An empty array is still SQL `NULL`, so existing `ANY($1)` callers stay unchanged. The setup statements already cast `$1` to `text[]`. - Integration tests for the username and org-domain statements pass an empty cursor and a page size. - `TestTextArray_Value` expects quoted elements, and a round trip through `Scan` covers a comma, a quote, and a backslash. - initial fix https://github.com/zitadel/zitadel/pull/12740 - fix of the setup job, broken by the first fix https://github.com/zitadel/zitadel/pull/12785 - needs to be a `fix` tag because `perf` wouldn't bump the version - Based on `owner-app-fix`. - A setup process already running the old statement keeps that plan until it is restarted.
Open in Web Open in Cursor 
--------- Co-authored-by: Cursor Agent Co-authored-by: Silvan --- cmd/setup/79.go | 32 +++++++++--- cmd/setup/79/01_usernames_org_scoped.sql | 34 ++++++++++--- cmd/setup/79/02_usernames.sql | 48 +++++++++++++----- cmd/setup/79/03_external_idps.sql | 34 ++++++++++--- cmd/setup/79/04_org_name.sql | 34 ++++++++++--- cmd/setup/79/05_org_domain.sql | 36 ++++++++++--- cmd/setup/79/06_project_names.sql | 34 ++++++++++--- cmd/setup/79/07_appname.sql | 44 +++++++++++----- cmd/setup/79/08_project_role.sql | 34 ++++++++++--- cmd/setup/79/09_entity_ids.sql | 50 ++++++++++++++----- cmd/setup/79/10_project_grant.sql | 34 ++++++++++--- cmd/setup/79/11_project_grant_member.sql | 40 +++++++++++---- cmd/setup/79/12_user_grant.sql | 40 +++++++++++---- cmd/setup/79/13_member_org.sql | 34 ++++++++++--- cmd/setup/79/14_member_project.sql | 34 ++++++++++--- cmd/setup/79/15_member_instance.sql | 34 ++++++++++--- cmd/setup/79/17_action_names.sql | 34 ++++++++++--- cmd/setup/79/18_idp_config_names.sql | 34 ++++++++++--- .../79/19_idp_config_names_templates.sql | 34 ++++++++++--- cmd/setup/79/20_mail_text.sql | 36 ++++++++++--- cmd/setup/79_test.go | 10 +++- .../unique_constraint_owners_test.go | 4 +- cmd/setup/setup.go | 7 ++- cmd/setup/steps.yaml | 2 + internal/database/type.go | 22 ++++++-- internal/database/type_test.go | 13 ++++- 26 files changed, 620 insertions(+), 172 deletions(-) diff --git a/cmd/setup/79.go b/cmd/setup/79.go index eca7b3836f..d2a6421c64 100644 --- a/cmd/setup/79.go +++ b/cmd/setup/79.go @@ -5,6 +5,8 @@ import ( "database/sql" "embed" "errors" + "fmt" + "slices" "strings" "github.com/jackc/pgx/v5/pgconn" @@ -26,6 +28,7 @@ type BackfillUniqueConstraintOwners struct { Version string `json:"version"` Finalized bool `json:"finalized"` ForceFinalize bool `json:"-"` // YAML/env only; not persisted on lastRun + BatchSize uint16 `json:"-"` // YAML/env only; not persisted on lastRun lastVersion string `json:"-"` lastFinalized bool `json:"-"` @@ -55,13 +58,28 @@ func (mig *BackfillUniqueConstraintOwners) Execute(ctx context.Context, _ events return err } for _, stmt := range statements { - logging.Info(ctx, "backfill unique constraint owners", "file", stmt.file, "migration", mig.String()) - if _, err := mig.dbClient.ExecContext(ctx, stmt.query); err != nil { - if isUndefinedTable(err) { - logging.Info(ctx, "skip unique constraint owners backfill, relation missing", "file", stmt.file, "migration", mig.String()) - continue + var cursor database.TextArray[string] + for page := 1; ; page++ { + var next database.TextArray[string] + var updated int64 + err := mig.dbClient.QueryRowContext(ctx, func(row *sql.Row) error { + return row.Scan(&next, &updated) + }, stmt.query, cursor, mig.BatchSize) + if err != nil { + if isUndefinedTable(err) { + logging.Info(ctx, "skip unique constraint owners backfill, relation missing", "file", stmt.file, "migration", mig.String()) + break + } + return fmt.Errorf("backfill unique constraint owners %s: %w", stmt.file, err) } - return err + if len(next) == 0 { + break + } + if slices.Equal(cursor, next) { + return fmt.Errorf("backfill unique constraint owners %s: cursor did not advance", stmt.file) + } + logging.Info(ctx, "backfill unique constraint owners page", "file", stmt.file, "page", page, "updated", updated, "migration", mig.String()) + cursor = next } } @@ -123,6 +141,8 @@ func backfillUniqueConstraintOwnersStatements() ([]statement, error) { } // BackfillUniqueConstraintOwnersQuery returns the interpolated SQL for a setup 79 file. +// The statement reads one projection page: $1 is a text[] primary-key cursor (nil starts at the beginning) and $2 is the page size. +// It returns the next cursor and the number of unique-constraint rows updated. func BackfillUniqueConstraintOwnersQuery(file string) (string, error) { statements, err := backfillUniqueConstraintOwnersStatements() if err != nil { diff --git a/cmd/setup/79/01_usernames_org_scoped.sql b/cmd/setup/79/01_usernames_org_scoped.sql index 1bbc728a79..510c8f42f7 100644 --- a/cmd/setup/79/01_usernames_org_scoped.sql +++ b/cmd/setup/79/01_usernames_org_scoped.sql @@ -1,8 +1,28 @@ -- unique_field: username+resourceOwner (user.NewAddUsernameUniqueConstraint, org-scoped) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || u.resource_owner, 'user:' || u.id] -FROM {{.users}} u -WHERE uc.unique_type = 'usernames' - AND uc.instance_id = u.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (u.username || u.resource_owner, lower(u.username || u.resource_owner)); +WITH page AS MATERIALIZED ( + SELECT u.instance_id, u.id, u.resource_owner, u.username + FROM {{.users}} u + WHERE (u.instance_id, u.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY u.instance_id, u.id + LIMIT $2 +), +keys AS ( + SELECT u.instance_id, u.id, u.resource_owner, u.username || u.resource_owner AS unique_field + FROM page u + UNION + SELECT u.instance_id, u.id, u.resource_owner, lower(u.username || u.resource_owner) + FROM page u +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || u.resource_owner, 'user:' || u.id] + FROM keys u + WHERE uc.instance_id = u.instance_id + AND uc.unique_type = 'usernames' + AND uc.unique_field = u.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/02_usernames.sql b/cmd/setup/79/02_usernames.sql index 5acf0e926b..a3cdbdb575 100644 --- a/cmd/setup/79/02_usernames.sql +++ b/cmd/setup/79/02_usernames.sql @@ -1,13 +1,37 @@ -- unique_field: username (user.NewAddUsernameUniqueConstraint, instance-scoped) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || u.resource_owner, 'user:' || u.id] -FROM {{.users}} u -LEFT JOIN {{.domain_policies}} org_pol - ON org_pol.instance_id = u.instance_id AND org_pol.id = u.resource_owner -LEFT JOIN {{.domain_policies}} inst_pol - ON inst_pol.instance_id = u.instance_id AND inst_pol.is_default -WHERE uc.unique_type = 'usernames' - AND uc.instance_id = u.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (u.username, lower(u.username)) - AND NOT COALESCE(org_pol.user_login_must_be_domain, inst_pol.user_login_must_be_domain, false); +WITH page AS MATERIALIZED ( + SELECT u.instance_id, u.id, u.resource_owner, u.username + FROM {{.users}} u + WHERE (u.instance_id, u.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY u.instance_id, u.id + LIMIT $2 +), +batch AS ( + SELECT u.instance_id, u.id, u.resource_owner, u.username + FROM page u + LEFT JOIN {{.domain_policies}} org_pol + ON org_pol.instance_id = u.instance_id AND org_pol.id = u.resource_owner + LEFT JOIN {{.domain_policies}} inst_pol + ON inst_pol.instance_id = u.instance_id AND inst_pol.is_default + WHERE NOT COALESCE(org_pol.user_login_must_be_domain, inst_pol.user_login_must_be_domain, false) +), +keys AS ( + SELECT u.instance_id, u.id, u.resource_owner, u.username AS unique_field + FROM batch u + UNION + SELECT u.instance_id, u.id, u.resource_owner, lower(u.username) + FROM batch u +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || u.resource_owner, 'user:' || u.id] + FROM keys u + WHERE uc.instance_id = u.instance_id + AND uc.unique_type = 'usernames' + AND uc.unique_field = u.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/03_external_idps.sql b/cmd/setup/79/03_external_idps.sql index 5a8c39e264..eccf0bb305 100644 --- a/cmd/setup/79/03_external_idps.sql +++ b/cmd/setup/79/03_external_idps.sql @@ -1,8 +1,28 @@ -- unique_field: idpConfigID+externalUserID (user.NewAddUserIDPLinkUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || l.resource_owner, 'idp:' || l.idp_id, 'user:' || l.user_id] -FROM {{.idp_user_links}} l -WHERE uc.unique_type = 'external_idps' - AND uc.instance_id = l.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (l.idp_id || l.external_user_id, lower(l.idp_id || l.external_user_id)); +WITH page AS MATERIALIZED ( + SELECT l.instance_id, l.idp_id, l.external_user_id, l.resource_owner, l.user_id + FROM {{.idp_user_links}} l + WHERE (l.instance_id, l.idp_id, l.external_user_id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], ''), COALESCE(($1::text[])[3], '')) + ORDER BY l.instance_id, l.idp_id, l.external_user_id + LIMIT $2 +), +keys AS ( + SELECT l.instance_id, l.resource_owner, l.idp_id, l.user_id, l.idp_id || l.external_user_id AS unique_field + FROM page l + UNION + SELECT l.instance_id, l.resource_owner, l.idp_id, l.user_id, lower(l.idp_id || l.external_user_id) + FROM page l +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || l.resource_owner, 'idp:' || l.idp_id, 'user:' || l.user_id] + FROM keys l + WHERE uc.instance_id = l.instance_id + AND uc.unique_type = 'external_idps' + AND uc.unique_field = l.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, idp_id, external_user_id] FROM page ORDER BY instance_id DESC, idp_id DESC, external_user_id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/04_org_name.sql b/cmd/setup/79/04_org_name.sql index baef09e28f..898c66e0a5 100644 --- a/cmd/setup/79/04_org_name.sql +++ b/cmd/setup/79/04_org_name.sql @@ -1,8 +1,28 @@ -- unique_field: org name (org.NewAddOrgNameUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || o.id] -FROM {{.orgs}} o -WHERE uc.unique_type = 'org_name' - AND uc.instance_id = o.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (o.name, lower(o.name)); +WITH page AS MATERIALIZED ( + SELECT o.instance_id, o.id, o.name + FROM {{.orgs}} o + WHERE (o.instance_id, o.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY o.instance_id, o.id + LIMIT $2 +), +keys AS ( + SELECT o.instance_id, o.id, o.name AS unique_field + FROM page o + UNION + SELECT o.instance_id, o.id, lower(o.name) + FROM page o +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || k.id] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'org_name' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/05_org_domain.sql b/cmd/setup/79/05_org_domain.sql index 997eed8d0b..3e25652ce3 100644 --- a/cmd/setup/79/05_org_domain.sql +++ b/cmd/setup/79/05_org_domain.sql @@ -1,9 +1,29 @@ -- unique_field: domain (org.NewAddOrgDomainUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || d.org_id] -FROM {{.org_domains}} d -WHERE uc.unique_type = 'org_domain' - AND uc.instance_id = d.instance_id - AND uc.owners = '{}' - AND d.is_verified - AND uc.unique_field IN (d.domain, lower(d.domain)); +WITH page AS MATERIALIZED ( + SELECT d.org_id, d.domain, d.instance_id + FROM {{.org_domains}} d + WHERE (d.org_id, d.domain, d.instance_id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], ''), COALESCE(($1::text[])[3], '')) + AND d.is_verified + ORDER BY d.org_id, d.domain, d.instance_id + LIMIT $2 +), +keys AS ( + SELECT d.instance_id, d.org_id, d.domain AS unique_field + FROM page d + UNION + SELECT d.instance_id, d.org_id, lower(d.domain) + FROM page d +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || k.org_id] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'org_domain' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[org_id, domain, instance_id] FROM page ORDER BY org_id DESC, domain DESC, instance_id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/06_project_names.sql b/cmd/setup/79/06_project_names.sql index ce7a49dd20..50bdbc8480 100644 --- a/cmd/setup/79/06_project_names.sql +++ b/cmd/setup/79/06_project_names.sql @@ -1,8 +1,28 @@ -- unique_field: name+resourceOwner (project.NewAddProjectNameUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || p.resource_owner, 'project:' || p.id] -FROM {{.projects}} p -WHERE uc.unique_type = 'project_names' - AND uc.instance_id = p.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (p.name || p.resource_owner, lower(p.name || p.resource_owner)); +WITH page AS MATERIALIZED ( + SELECT p.instance_id, p.id, p.resource_owner, p.name + FROM {{.projects}} p + WHERE (p.instance_id, p.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY p.instance_id, p.id + LIMIT $2 +), +keys AS ( + SELECT p.instance_id, p.id, p.resource_owner, p.name || p.resource_owner AS unique_field + FROM page p + UNION + SELECT p.instance_id, p.id, p.resource_owner, lower(p.name || p.resource_owner) + FROM page p +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || k.resource_owner, 'project:' || k.id] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'project_names' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/07_appname.sql b/cmd/setup/79/07_appname.sql index ec58148eb8..8a6ae523e6 100644 --- a/cmd/setup/79/07_appname.sql +++ b/cmd/setup/79/07_appname.sql @@ -1,13 +1,33 @@ -- unique_field: name:projectID (project.NewAddApplicationUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ( - CASE WHEN uc.owners = '{}' - THEN ARRAY['org:' || a.resource_owner, 'project:' || a.project_id] - ELSE uc.owners - END -) || ARRAY['app:' || a.id] -FROM {{.apps}} a -WHERE uc.unique_type = 'appname' - AND uc.instance_id = a.instance_id - AND NOT (uc.owners @> ARRAY['app:' || a.id]) - AND uc.unique_field IN (a.name || ':' || a.project_id, lower(a.name || ':' || a.project_id)); +WITH page AS MATERIALIZED ( + SELECT a.instance_id, a.id, a.resource_owner, a.project_id, a.name + FROM {{.apps}} a + WHERE (a.instance_id, a.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY a.instance_id, a.id + LIMIT $2 +), +keys AS ( + SELECT a.instance_id, a.id, a.resource_owner, a.project_id, a.name || ':' || a.project_id AS unique_field + FROM page a + UNION + SELECT a.instance_id, a.id, a.resource_owner, a.project_id, lower(a.name || ':' || a.project_id) + FROM page a +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ( + CASE WHEN uc.owners = '{}' + THEN ARRAY['org:' || a.resource_owner, 'project:' || a.project_id] + ELSE uc.owners + END + ) || ARRAY['app:' || a.id] + FROM keys a + WHERE uc.instance_id = a.instance_id + AND uc.unique_type = 'appname' + AND uc.unique_field = a.unique_field + AND NOT (uc.owners @> ARRAY['app:' || a.id]) + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/08_project_role.sql b/cmd/setup/79/08_project_role.sql index 394d752add..f01cab7a44 100644 --- a/cmd/setup/79/08_project_role.sql +++ b/cmd/setup/79/08_project_role.sql @@ -1,8 +1,28 @@ -- unique_field: roleKey:projectID (project.NewAddProjectRoleUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || r.resource_owner, 'project:' || r.project_id] -FROM {{.project_roles}} r -WHERE uc.unique_type = 'project_role' - AND uc.instance_id = r.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (r.role_key || ':' || r.project_id, lower(r.role_key || ':' || r.project_id)); +WITH page AS MATERIALIZED ( + SELECT r.instance_id, r.project_id, r.role_key, r.resource_owner + FROM {{.project_roles}} r + WHERE (r.instance_id, r.project_id, r.role_key) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], ''), COALESCE(($1::text[])[3], '')) + ORDER BY r.instance_id, r.project_id, r.role_key + LIMIT $2 +), +keys AS ( + SELECT r.instance_id, r.resource_owner, r.project_id, r.role_key || ':' || r.project_id AS unique_field + FROM page r + UNION + SELECT r.instance_id, r.resource_owner, r.project_id, lower(r.role_key || ':' || r.project_id) + FROM page r +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || k.resource_owner, 'project:' || k.project_id] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'project_role' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, project_id, role_key] FROM page ORDER BY instance_id DESC, project_id DESC, role_key DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/09_entity_ids.sql b/cmd/setup/79/09_entity_ids.sql index f0394eb9ae..7c23b03aaa 100644 --- a/cmd/setup/79/09_entity_ids.sql +++ b/cmd/setup/79/09_entity_ids.sql @@ -1,14 +1,38 @@ -- unique_field: entityID (project.NewAddSAMLConfigEntityIDUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ( - CASE WHEN uc.owners = '{}' - THEN ARRAY['org:' || a.resource_owner, 'project:' || a.project_id] - ELSE uc.owners - END -) || ARRAY['app:' || a.id] -FROM {{.apps}} a -JOIN {{.apps_saml}} s ON s.app_id = a.id AND s.instance_id = a.instance_id -WHERE uc.unique_type = 'entity_ids' - AND uc.instance_id = a.instance_id - AND NOT (uc.owners @> ARRAY['app:' || a.id]) - AND uc.unique_field IN (s.entity_id, lower(s.entity_id)); +WITH page AS MATERIALIZED ( + SELECT s.instance_id, s.app_id, s.entity_id + FROM {{.apps_saml}} s + WHERE (s.instance_id, s.app_id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY s.instance_id, s.app_id + LIMIT $2 +), +batch AS ( + SELECT s.instance_id, a.id, a.resource_owner, a.project_id, s.entity_id + FROM page s + JOIN {{.apps}} a ON a.instance_id = s.instance_id AND a.id = s.app_id +), +keys AS ( + SELECT a.instance_id, a.id, a.resource_owner, a.project_id, a.entity_id AS unique_field + FROM batch a + UNION + SELECT a.instance_id, a.id, a.resource_owner, a.project_id, lower(a.entity_id) + FROM batch a +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ( + CASE WHEN uc.owners = '{}' + THEN ARRAY['org:' || a.resource_owner, 'project:' || a.project_id] + ELSE uc.owners + END + ) || ARRAY['app:' || a.id] + FROM keys a + WHERE uc.instance_id = a.instance_id + AND uc.unique_type = 'entity_ids' + AND uc.unique_field = a.unique_field + AND NOT (uc.owners @> ARRAY['app:' || a.id]) + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, app_id] FROM page ORDER BY instance_id DESC, app_id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/10_project_grant.sql b/cmd/setup/79/10_project_grant.sql index e08a8d1abb..0989fa54a1 100644 --- a/cmd/setup/79/10_project_grant.sql +++ b/cmd/setup/79/10_project_grant.sql @@ -1,9 +1,29 @@ -- unique_field: grantedOrgID:projectID (project.NewAddProjectGrantUniqueConstraint) -- resource_owner is the project org and granted_org_id is the granted org; they differ -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || g.resource_owner, 'org:' || g.granted_org_id, 'project:' || g.project_id, 'grant:' || g.grant_id] -FROM {{.project_grants}} g -WHERE uc.unique_type = 'project_grant' - AND uc.instance_id = g.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (g.granted_org_id || ':' || g.project_id, lower(g.granted_org_id || ':' || g.project_id)); +WITH page AS MATERIALIZED ( + SELECT g.instance_id, g.grant_id, g.resource_owner, g.granted_org_id, g.project_id + FROM {{.project_grants}} g + WHERE (g.instance_id, g.grant_id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY g.instance_id, g.grant_id + LIMIT $2 +), +keys AS ( + SELECT g.instance_id, g.resource_owner, g.granted_org_id, g.project_id, g.grant_id, g.granted_org_id || ':' || g.project_id AS unique_field + FROM page g + UNION + SELECT g.instance_id, g.resource_owner, g.granted_org_id, g.project_id, g.grant_id, lower(g.granted_org_id || ':' || g.project_id) + FROM page g +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || g.resource_owner, 'org:' || g.granted_org_id, 'project:' || g.project_id, 'grant:' || g.grant_id] + FROM keys g + WHERE uc.instance_id = g.instance_id + AND uc.unique_type = 'project_grant' + AND uc.unique_field = g.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, grant_id] FROM page ORDER BY instance_id DESC, grant_id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/11_project_grant_member.sql b/cmd/setup/79/11_project_grant_member.sql index c75007af0b..a755024d20 100644 --- a/cmd/setup/79/11_project_grant_member.sql +++ b/cmd/setup/79/11_project_grant_member.sql @@ -1,11 +1,33 @@ -- unique_field: projectID:userID:grantID (project.NewAddProjectGrantMemberUniqueConstraint) -- resource_owner is the project org; also stamp the user's home org and the granted org -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || m.resource_owner, 'org:' || m.user_resource_owner, 'user:' || m.user_id, 'project:' || m.project_id] - || CASE WHEN COALESCE(m.granted_org, '') <> '' THEN ARRAY['org:' || m.granted_org] ELSE ARRAY[]::text[] END - || CASE WHEN COALESCE(m.grant_id, '') <> '' THEN ARRAY['grant:' || m.grant_id] ELSE ARRAY[]::text[] END -FROM {{.project_grant_members}} m -WHERE uc.unique_type = 'project_grant_member' - AND uc.instance_id = m.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (m.project_id || ':' || m.user_id || ':' || m.grant_id, lower(m.project_id || ':' || m.user_id || ':' || m.grant_id)); +WITH page AS MATERIALIZED ( + SELECT m.instance_id, m.project_id, m.grant_id, m.user_id, m.resource_owner, m.user_resource_owner, m.granted_org + FROM {{.project_grant_members}} m + WHERE (m.instance_id, m.project_id, m.grant_id, m.user_id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], ''), COALESCE(($1::text[])[3], ''), COALESCE(($1::text[])[4], '')) + ORDER BY m.instance_id, m.project_id, m.grant_id, m.user_id + LIMIT $2 +), +keys AS ( + SELECT m.instance_id, m.resource_owner, m.user_resource_owner, m.user_id, m.project_id, m.granted_org, m.grant_id, + m.project_id || ':' || m.user_id || ':' || m.grant_id AS unique_field + FROM page m + UNION + SELECT m.instance_id, m.resource_owner, m.user_resource_owner, m.user_id, m.project_id, m.granted_org, m.grant_id, + lower(m.project_id || ':' || m.user_id || ':' || m.grant_id) + FROM page m +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || m.resource_owner, 'org:' || m.user_resource_owner, 'user:' || m.user_id, 'project:' || m.project_id] + || CASE WHEN COALESCE(m.granted_org, '') <> '' THEN ARRAY['org:' || m.granted_org] ELSE ARRAY[]::text[] END + || CASE WHEN COALESCE(m.grant_id, '') <> '' THEN ARRAY['grant:' || m.grant_id] ELSE ARRAY[]::text[] END + FROM keys m + WHERE uc.instance_id = m.instance_id + AND uc.unique_type = 'project_grant_member' + AND uc.unique_field = m.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, project_id, grant_id, user_id] FROM page ORDER BY instance_id DESC, project_id DESC, grant_id DESC, user_id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/12_user_grant.sql b/cmd/setup/79/12_user_grant.sql index 1e47f329f4..27ed9a9535 100644 --- a/cmd/setup/79/12_user_grant.sql +++ b/cmd/setup/79/12_user_grant.sql @@ -1,11 +1,33 @@ -- unique_field: resourceOwner:userID:projectID:grantID (usergrant.NewAddUserGrantUniqueConstraint) -- grant resource_owner is not always the user's org; stamp user, project, and granted orgs -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || g.resource_owner, 'org:' || g.resource_owner_user, 'org:' || g.resource_owner_project, 'user:' || g.user_id, 'project:' || g.project_id] - || CASE WHEN COALESCE(g.granted_org, '') <> '' THEN ARRAY['org:' || g.granted_org] ELSE ARRAY[]::text[] END - || CASE WHEN COALESCE(g.grant_id, '') <> '' THEN ARRAY['grant:' || g.grant_id] ELSE ARRAY[]::text[] END -FROM {{.user_grants}} g -WHERE uc.unique_type = 'user_grant' - AND uc.instance_id = g.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (g.resource_owner || ':' || g.user_id || ':' || g.project_id || ':' || g.grant_id, lower(g.resource_owner || ':' || g.user_id || ':' || g.project_id || ':' || g.grant_id)); +WITH page AS MATERIALIZED ( + SELECT g.instance_id, g.id, g.resource_owner, g.resource_owner_user, g.resource_owner_project, g.user_id, g.project_id, g.granted_org, g.grant_id + FROM {{.user_grants}} g + WHERE (g.instance_id, g.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY g.instance_id, g.id + LIMIT $2 +), +keys AS ( + SELECT g.instance_id, g.resource_owner, g.resource_owner_user, g.resource_owner_project, g.user_id, g.project_id, g.granted_org, g.grant_id, + g.resource_owner || ':' || g.user_id || ':' || g.project_id || ':' || g.grant_id AS unique_field + FROM page g + UNION + SELECT g.instance_id, g.resource_owner, g.resource_owner_user, g.resource_owner_project, g.user_id, g.project_id, g.granted_org, g.grant_id, + lower(g.resource_owner || ':' || g.user_id || ':' || g.project_id || ':' || g.grant_id) + FROM page g +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || g.resource_owner, 'org:' || g.resource_owner_user, 'org:' || g.resource_owner_project, 'user:' || g.user_id, 'project:' || g.project_id] + || CASE WHEN COALESCE(g.granted_org, '') <> '' THEN ARRAY['org:' || g.granted_org] ELSE ARRAY[]::text[] END + || CASE WHEN COALESCE(g.grant_id, '') <> '' THEN ARRAY['grant:' || g.grant_id] ELSE ARRAY[]::text[] END + FROM keys g + WHERE uc.instance_id = g.instance_id + AND uc.unique_type = 'user_grant' + AND uc.unique_field = g.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/13_member_org.sql b/cmd/setup/79/13_member_org.sql index a3c143c121..f3b23d9229 100644 --- a/cmd/setup/79/13_member_org.sql +++ b/cmd/setup/79/13_member_org.sql @@ -1,9 +1,29 @@ -- unique_field: orgID:userID (member.NewAddMemberUniqueConstraint, org aggregate) -- stamp both the membership org and the user's home org (they differ for cross-org members) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || m.org_id, 'org:' || m.user_resource_owner, 'user:' || m.user_id] -FROM {{.org_members}} m -WHERE uc.unique_type = 'member' - AND uc.instance_id = m.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (m.org_id || ':' || m.user_id, lower(m.org_id || ':' || m.user_id)); +WITH page AS MATERIALIZED ( + SELECT m.instance_id, m.org_id, m.user_id, m.user_resource_owner + FROM {{.org_members}} m + WHERE (m.instance_id, m.org_id, m.user_id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], ''), COALESCE(($1::text[])[3], '')) + ORDER BY m.instance_id, m.org_id, m.user_id + LIMIT $2 +), +keys AS ( + SELECT m.instance_id, m.org_id, m.user_resource_owner, m.user_id, m.org_id || ':' || m.user_id AS unique_field + FROM page m + UNION + SELECT m.instance_id, m.org_id, m.user_resource_owner, m.user_id, lower(m.org_id || ':' || m.user_id) + FROM page m +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || k.org_id, 'org:' || k.user_resource_owner, 'user:' || k.user_id] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'member' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, org_id, user_id] FROM page ORDER BY instance_id DESC, org_id DESC, user_id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/14_member_project.sql b/cmd/setup/79/14_member_project.sql index 1dce4c8d48..b2fbcec05c 100644 --- a/cmd/setup/79/14_member_project.sql +++ b/cmd/setup/79/14_member_project.sql @@ -1,9 +1,29 @@ -- unique_field: projectID:userID (member.NewAddMemberUniqueConstraint, project aggregate) -- stamp the project org and the user's home org (they differ for cross-org members) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || m.resource_owner, 'org:' || m.user_resource_owner, 'user:' || m.user_id, 'project:' || m.project_id] -FROM {{.project_members}} m -WHERE uc.unique_type = 'member' - AND uc.instance_id = m.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (m.project_id || ':' || m.user_id, lower(m.project_id || ':' || m.user_id)); +WITH page AS MATERIALIZED ( + SELECT m.instance_id, m.project_id, m.user_id, m.resource_owner, m.user_resource_owner + FROM {{.project_members}} m + WHERE (m.instance_id, m.project_id, m.user_id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], ''), COALESCE(($1::text[])[3], '')) + ORDER BY m.instance_id, m.project_id, m.user_id + LIMIT $2 +), +keys AS ( + SELECT m.instance_id, m.resource_owner, m.user_resource_owner, m.user_id, m.project_id, m.project_id || ':' || m.user_id AS unique_field + FROM page m + UNION + SELECT m.instance_id, m.resource_owner, m.user_resource_owner, m.user_id, m.project_id, lower(m.project_id || ':' || m.user_id) + FROM page m +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || k.resource_owner, 'org:' || k.user_resource_owner, 'user:' || k.user_id, 'project:' || k.project_id] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'member' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, project_id, user_id] FROM page ORDER BY instance_id DESC, project_id DESC, user_id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/15_member_instance.sql b/cmd/setup/79/15_member_instance.sql index f2625e93f0..23ace15d22 100644 --- a/cmd/setup/79/15_member_instance.sql +++ b/cmd/setup/79/15_member_instance.sql @@ -1,9 +1,29 @@ -- unique_field: instanceID:userID (member.NewAddMemberUniqueConstraint, instance aggregate) -- instance members live on the instance aggregate, so also stamp the user's home org -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['user:' || m.user_id, 'org:' || m.user_resource_owner] -FROM {{.instance_members}} m -WHERE uc.unique_type = 'member' - AND uc.instance_id = m.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (m.id || ':' || m.user_id, lower(m.id || ':' || m.user_id)); +WITH page AS MATERIALIZED ( + SELECT m.instance_id, m.id, m.user_id, m.user_resource_owner + FROM {{.instance_members}} m + WHERE (m.instance_id, m.id, m.user_id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], ''), COALESCE(($1::text[])[3], '')) + ORDER BY m.instance_id, m.id, m.user_id + LIMIT $2 +), +keys AS ( + SELECT m.instance_id, m.user_id, m.user_resource_owner, m.id || ':' || m.user_id AS unique_field + FROM page m + UNION + SELECT m.instance_id, m.user_id, m.user_resource_owner, lower(m.id || ':' || m.user_id) + FROM page m +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['user:' || k.user_id, 'org:' || k.user_resource_owner] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'member' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id, user_id] FROM page ORDER BY instance_id DESC, id DESC, user_id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/17_action_names.sql b/cmd/setup/79/17_action_names.sql index 62ddb63277..6254f66df9 100644 --- a/cmd/setup/79/17_action_names.sql +++ b/cmd/setup/79/17_action_names.sql @@ -1,8 +1,28 @@ -- unique_field: name:resourceOwner (action.NewAddActionNameUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || a.resource_owner] -FROM {{.actions}} a -WHERE uc.unique_type = 'action_names' - AND uc.instance_id = a.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (a.name || ':' || a.resource_owner, lower(a.name || ':' || a.resource_owner)); +WITH page AS MATERIALIZED ( + SELECT a.instance_id, a.id, a.resource_owner, a.name + FROM {{.actions}} a + WHERE (a.instance_id, a.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY a.instance_id, a.id + LIMIT $2 +), +keys AS ( + SELECT a.instance_id, a.resource_owner, a.name || ':' || a.resource_owner AS unique_field + FROM page a + UNION + SELECT a.instance_id, a.resource_owner, lower(a.name || ':' || a.resource_owner) + FROM page a +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || k.resource_owner] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'action_names' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/18_idp_config_names.sql b/cmd/setup/79/18_idp_config_names.sql index 3b35e5bc10..e772447a87 100644 --- a/cmd/setup/79/18_idp_config_names.sql +++ b/cmd/setup/79/18_idp_config_names.sql @@ -1,8 +1,28 @@ -- unique_field: name+resourceOwner (idpconfig.NewAddIDPConfigNameUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || i.resource_owner, 'idp:' || i.id] -FROM {{.idps}} i -WHERE uc.unique_type = 'idp_config_names' - AND uc.instance_id = i.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (i.name || i.resource_owner, lower(i.name || i.resource_owner)); +WITH page AS MATERIALIZED ( + SELECT i.instance_id, i.id, i.resource_owner, i.name + FROM {{.idps}} i + WHERE (i.instance_id, i.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY i.instance_id, i.id + LIMIT $2 +), +keys AS ( + SELECT i.instance_id, i.resource_owner, i.id, i.name || i.resource_owner AS unique_field + FROM page i + UNION + SELECT i.instance_id, i.resource_owner, i.id, lower(i.name || i.resource_owner) + FROM page i +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || i.resource_owner, 'idp:' || i.id] + FROM keys i + WHERE uc.instance_id = i.instance_id + AND uc.unique_type = 'idp_config_names' + AND uc.unique_field = i.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/19_idp_config_names_templates.sql b/cmd/setup/79/19_idp_config_names_templates.sql index f1cae9ddaa..f364f1499f 100644 --- a/cmd/setup/79/19_idp_config_names_templates.sql +++ b/cmd/setup/79/19_idp_config_names_templates.sql @@ -1,8 +1,28 @@ -- unique_field: name+resourceOwner (idpconfig.NewAddIDPConfigNameUniqueConstraint, templates) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || t.resource_owner, 'idp:' || t.id] -FROM {{.idp_templates}} t -WHERE uc.unique_type = 'idp_config_names' - AND uc.instance_id = t.instance_id - AND uc.owners = '{}' - AND uc.unique_field IN (t.name || t.resource_owner, lower(t.name || t.resource_owner)); +WITH page AS MATERIALIZED ( + SELECT t.instance_id, t.id, t.resource_owner, t.name + FROM {{.idp_templates}} t + WHERE (t.instance_id, t.id) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], '')) + ORDER BY t.instance_id, t.id + LIMIT $2 +), +keys AS ( + SELECT t.instance_id, t.resource_owner, t.id, t.name || t.resource_owner AS unique_field + FROM page t + UNION + SELECT t.instance_id, t.resource_owner, t.id, lower(t.name || t.resource_owner) + FROM page t +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || t.resource_owner, 'idp:' || t.id] + FROM keys t + WHERE uc.instance_id = t.instance_id + AND uc.unique_type = 'idp_config_names' + AND uc.unique_field = t.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, id] FROM page ORDER BY instance_id DESC, id DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79/20_mail_text.sql b/cmd/setup/79/20_mail_text.sql index 9725c85c78..e3343a3496 100644 --- a/cmd/setup/79/20_mail_text.sql +++ b/cmd/setup/79/20_mail_text.sql @@ -1,9 +1,29 @@ -- unique_field: aggregateID:mailTextType:language (policy.NewAddMailTextUniqueConstraint) -UPDATE eventstore.unique_constraints uc -SET owners = ARRAY['org:' || t.aggregate_id] -FROM {{.message_texts}} t -WHERE uc.unique_type = 'mail_text' - AND uc.instance_id = t.instance_id - AND uc.owners = '{}' - AND t.aggregate_id <> t.instance_id - AND uc.unique_field IN (t.aggregate_id || ':' || t.type || ':' || t.language, lower(t.aggregate_id || ':' || t.type || ':' || t.language)); +WITH page AS MATERIALIZED ( + SELECT t.instance_id, t.aggregate_id, t.type, t.language + FROM {{.message_texts}} t + WHERE (t.instance_id, t.aggregate_id, t.type, t.language) > (COALESCE(($1::text[])[1], ''), COALESCE(($1::text[])[2], ''), COALESCE(($1::text[])[3], ''), COALESCE(($1::text[])[4], '')) + AND t.aggregate_id <> t.instance_id + ORDER BY t.instance_id, t.aggregate_id, t.type, t.language + LIMIT $2 +), +keys AS ( + SELECT t.instance_id, t.aggregate_id, t.aggregate_id || ':' || t.type || ':' || t.language AS unique_field + FROM page t + UNION + SELECT t.instance_id, t.aggregate_id, lower(t.aggregate_id || ':' || t.type || ':' || t.language) + FROM page t +), +upd AS ( + UPDATE eventstore.unique_constraints uc + SET owners = ARRAY['org:' || k.aggregate_id] + FROM keys k + WHERE uc.instance_id = k.instance_id + AND uc.unique_type = 'mail_text' + AND uc.unique_field = k.unique_field + AND uc.owners = '{}' + RETURNING 1 +) +SELECT + (SELECT ARRAY[instance_id, aggregate_id, type, language] FROM page ORDER BY instance_id DESC, aggregate_id DESC, type DESC, language DESC LIMIT 1), + (SELECT COUNT(*) FROM upd); diff --git a/cmd/setup/79_test.go b/cmd/setup/79_test.go index 5ef876b6f6..366418e07c 100644 --- a/cmd/setup/79_test.go +++ b/cmd/setup/79_test.go @@ -27,7 +27,7 @@ func TestBackfillUniqueConstraintOwnersStmts(t *testing.T) { extra []string }{ {"01_usernames_org_scoped.sql", "usernames", "u.username || u.resource_owner", projection.UserTable, []string{"'org:' || u.resource_owner"}}, - {"02_usernames.sql", "usernames", "u.username, lower(u.username)", projection.UserTable, []string{"'org:' || u.resource_owner", "user_login_must_be_domain"}}, + {"02_usernames.sql", "usernames", "lower(u.username)", projection.UserTable, []string{"'org:' || u.resource_owner", "organization_scoped_usernames", "user_login_must_be_domain"}}, {"03_external_idps.sql", "external_idps", "l.idp_id || l.external_user_id", projection.IDPUserLinkTable, []string{"'idp:' || l.idp_id"}}, {"04_org_name.sql", "org_name", "o.name", projection.OrgProjectionTable, nil}, {"05_org_domain.sql", "org_domain", "d.domain", projection.OrgDomainTable, []string{"d.is_verified"}}, @@ -53,7 +53,10 @@ func TestBackfillUniqueConstraintOwnersStmts(t *testing.T) { assert.Contains(t, stmt.query, "unique_type = '"+want[i].uniqueType+"'") assert.Contains(t, stmt.query, want[i].field) assert.Contains(t, stmt.query, want[i].table) - assert.Contains(t, stmt.query, "uc.unique_field IN (") + assert.Contains(t, stmt.query, "uc.unique_field =") + assert.Contains(t, stmt.query, "LIMIT $2") + assert.Contains(t, stmt.query, "($1::text[])[1]") + assert.NotContains(t, stmt.query, "uc.unique_field IN (") assert.NotContains(t, stmt.query, "lower(uc.unique_field)") assert.NotContains(t, stmt.query, "{{.") for _, extra := range want[i].extra { @@ -194,6 +197,7 @@ func TestBackfillUniqueConstraintOwnersJSONOmitsForceFinalize(t *testing.T) { Version: "v2.0.0", Finalized: true, ForceFinalize: true, + BatchSize: 10, lastVersion: "v1.0.0", lastFinalized: true, } @@ -206,6 +210,8 @@ func TestBackfillUniqueConstraintOwnersJSONOmitsForceFinalize(t *testing.T) { assert.Equal(t, true, payload["finalized"]) assert.NotContains(t, payload, "ForceFinalize") assert.NotContains(t, payload, "forceFinalize") + assert.NotContains(t, payload, "BatchSize") + assert.NotContains(t, payload, "batchSize") assert.NotContains(t, payload, "lastVersion") assert.NotContains(t, payload, "lastFinalized") assert.False(t, strings.Contains(string(data), "v1.0.0")) diff --git a/cmd/setup/integration_test/unique_constraint_owners_test.go b/cmd/setup/integration_test/unique_constraint_owners_test.go index 64243d0e39..31ab4cbd58 100644 --- a/cmd/setup/integration_test/unique_constraint_owners_test.go +++ b/cmd/setup/integration_test/unique_constraint_owners_test.go @@ -33,7 +33,7 @@ func TestBackfillUniqueConstraintOwners_OrgDomainVerifiedOnly(t *testing.T) { query, err := setup.BackfillUniqueConstraintOwnersQuery("05_org_domain.sql") require.NoError(t, err) - _, err = dbPool.Exec(CTX, query) + _, err = dbPool.Exec(CTX, query, nil, 5000) require.NoError(t, err) var owners []string @@ -81,7 +81,7 @@ func TestBackfillUniqueConstraintOwners_UsernameInstanceScopedOrg(t *testing.T) query, err := setup.BackfillUniqueConstraintOwnersQuery("02_usernames.sql") require.NoError(t, err) - _, err = dbPool.Exec(CTX, query) + _, err = dbPool.Exec(CTX, query, nil, 5000) require.NoError(t, err) var instanceOwners []string diff --git a/cmd/setup/setup.go b/cmd/setup/setup.go index 12ec040d8d..092942e547 100644 --- a/cmd/setup/setup.go +++ b/cmd/setup/setup.go @@ -327,7 +327,6 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) steps.s76Users14LoginEqualityIndexes, steps.s77StampEventPositionAtInsert, steps.s78UniqueConstraintOwners, - steps.BackfillUniqueConstraintOwners, } { setupErr = executeMigration(ctx, eventstoreClient, step, "migration failed") if setupErr != nil { @@ -414,6 +413,12 @@ func Setup(ctx context.Context, config *Config, steps *Steps, masterKey string) return } } + + // Owner backfill joins projection rows, so it runs after those tables are created and after event replay when prefill is enabled. + setupErr = executeMigration(ctx, eventstoreClient, steps.BackfillUniqueConstraintOwners, "migration failed") + if setupErr != nil { + return setupErr + } return nil } diff --git a/cmd/setup/steps.yaml b/cmd/setup/steps.yaml index dcf55d481d..3622f81b88 100644 --- a/cmd/setup/steps.yaml +++ b/cmd/setup/steps.yaml @@ -187,3 +187,5 @@ BackfillUniqueConstraintOwners: # backfill leftover empty owners and enable owner-only unique-constraint delete. # Default false: owner-delete waits until the next release runs 79 again. ForceFinalize: false # ZITADEL_BACKFILLUNIQUECONSTRAINTOWNERS_FORCEFINALIZE + # Rows read from each projection per statement. Each page commits on its own. + BatchSize: 5000 # ZITADEL_BACKFILLUNIQUECONSTRAINTOWNERS_BATCHSIZE diff --git a/internal/database/type.go b/internal/database/type.go index bd07e0bfde..9d56976fcf 100644 --- a/internal/database/type.go +++ b/internal/database/type.go @@ -30,18 +30,30 @@ func (s *TextArray[T]) Scan(src any) error { } // Value implements the [database/sql/driver.Valuer] interface. +// Every element is quoted so commas, quotes, and backslashes stay inside one element. +// An empty array is NULL, which keeps ANY($1) filters unchanged. func (s TextArray[T]) Value() (driver.Value, error) { if len(s) == 0 { return nil, nil } - typed := make([]string, len(s)) - + var b strings.Builder + b.WriteByte('{') for i, value := range s { - typed[i] = string(value) + if i > 0 { + b.WriteByte(',') + } + b.WriteByte('"') + for _, r := range string(value) { + if r == '"' || r == '\\' { + b.WriteByte('\\') + } + b.WriteRune(r) + } + b.WriteByte('"') } - - return []byte("{" + strings.Join(typed, ",") + "}"), nil + b.WriteByte('}') + return []byte(b.String()), nil } type ByteArray[T ~byte] pgtype.FlatArray[T] diff --git a/internal/database/type_test.go b/internal/database/type_test.go index 7fab568a4e..a87069dc39 100644 --- a/internal/database/type_test.go +++ b/internal/database/type_test.go @@ -313,7 +313,7 @@ func TestTextArray_Value(t *testing.T) { "set", TextArray[string]{"a", "s", "d", "f"}, res{ - want: driver.Value([]byte("{a,s,d,f}")), + want: driver.Value([]byte(`{"a","s","d","f"}`)), }, }, } @@ -331,6 +331,17 @@ func TestTextArray_Value(t *testing.T) { } } +func TestTextArray_ValueScanRoundTrip(t *testing.T) { + original := TextArray[string]{"a,b", `say "hi"`, `a\b`} + + value, err := original.Value() + require.NoError(t, err) + + var scanned TextArray[string] + require.NoError(t, scanned.Scan(value)) + assert.Equal(t, original, scanned) +} + type typedByte byte func TestByteArray_Scan(t *testing.T) {