Files
zitadel/internal/database/type.go
T
25c23cf91d fix: setup step 79 batch unique constraint owner backfill (#12803)
<!-- CURSOR_AGENT_PR_BODY_BEGIN -->

- Setup 79 rewrites unique-constraint owners in one statement per file.
On a large table that statement holds row locks for the whole update,
and adding `LIMIT` does not help: the username statement bitmap-scans
every empty `owners` value and then index-scans every user in the
instance for each constraint.
- The backfill joined projection tables before those tables were created
and before event replay filled them.
- The page cursor is a `text[]`. `database.TextArray.Value` joined
elements with commas and never quoted them, so a project role key such
as `a,b` became two array elements. The next page repeated the same row
and the migration stopped.

- Each setup 79 statement pages its projection by primary key and
updates `eventstore.unique_constraints` with an equality lookup on
`(instance_id, unique_type, unique_field)`.
- The original field and `lower(...)` are separate `UNION` arms, so the
lookup stays a primary-key probe.
- Setup repeats a file until the projection page is empty. A page that
matches no constraint still advances the cursor.
- `BatchSize` is a `uint16`, configured like `ForceFinalize`
(`cmd/setup/steps.yaml`,
`ZITADEL_BACKFILLUNIQUECONSTRAINTOWNERS_BATCHSIZE`) and is not stored on
lastRun. The default in `steps.yaml` is 5000.
- Each page logs the file, the page index, and the number of rows
updated.
- The backfill runs after projection prefill. When `InitProjections` is
disabled or setup is for mirror, it still runs against the projection
rows already stored.
- `TextArray.Value` quotes every element and backslash-escapes `"` and
`\`. An empty array is still SQL `NULL`, so existing `ANY($1)` callers
stay unchanged. The setup statements already cast `$1` to `text[]`.

- Integration tests for the username and org-domain statements pass an
empty cursor and a page size.
- `TestTextArray_Value` expects quoted elements, and a round trip
through `Scan` covers a comma, a quote, and a backslash.
- initial fix https://github.com/zitadel/zitadel/pull/12740
- fix of the setup job, broken by the first fix
https://github.com/zitadel/zitadel/pull/12785
- needs to be a `fix` tag because `perf` wouldn't bump the version

- Based on `owner-app-fix`.
- A setup process already running the old statement keeps that plan
until it is restarted.

<!-- CURSOR_AGENT_PR_BODY_END -->

<div><a
href="https://cursor.com/agents/bc-31bfb5c4-5888-43f0-85cf-af1af097ca30?cursor_ref=pr_footer&cursor_cta=open_in_web"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://cursor.com/assets/images/open-in-web-dark.png"><source
media="(prefers-color-scheme: light)"
srcset="https://cursor.com/assets/images/open-in-web-light.png"><img
alt="Open in Web" width="114" height="28"
src="https://cursor.com/assets/images/open-in-web-dark.png"></picture></a>&nbsp;<a
href="https://cursor.com/background-agent?bcId=bc-31bfb5c4-5888-43f0-85cf-af1af097ca30&cursor_ref=pr_footer&cursor_cta=open_in_cursor"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://cursor.com/assets/images/open-in-cursor-dark.png"><source
media="(prefers-color-scheme: light)"
srcset="https://cursor.com/assets/images/open-in-cursor-light.png"><img
alt="Open in Cursor" width="131" height="28"
src="https://cursor.com/assets/images/open-in-cursor-dark.png"></picture></a>&nbsp;</div>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Silvan <adlerhurst@users.noreply.github.com>
2026-09-23 16:35:17 +02:00

274 lines
6.2 KiB
Go

package database
import (
"database/sql"
"database/sql/driver"
"encoding/json"
"reflect"
"strings"
"time"
"github.com/jackc/pgx/v5/pgtype"
)
type TextArray[T ~string] pgtype.FlatArray[T]
// Scan implements the [database/sql.Scanner] interface.
func (s *TextArray[T]) Scan(src any) error {
var typedArray []string
err := pgtype.NewMap().SQLScanner(&typedArray).Scan(src)
if err != nil {
return err
}
(*s) = make(TextArray[T], len(typedArray))
for i, value := range typedArray {
(*s)[i] = T(value)
}
return nil
}
// Value implements the [database/sql/driver.Valuer] interface.
// Every element is quoted so commas, quotes, and backslashes stay inside one element.
// An empty array is NULL, which keeps ANY($1) filters unchanged.
func (s TextArray[T]) Value() (driver.Value, error) {
if len(s) == 0 {
return nil, nil
}
var b strings.Builder
b.WriteByte('{')
for i, value := range s {
if i > 0 {
b.WriteByte(',')
}
b.WriteByte('"')
for _, r := range string(value) {
if r == '"' || r == '\\' {
b.WriteByte('\\')
}
b.WriteRune(r)
}
b.WriteByte('"')
}
b.WriteByte('}')
return []byte(b.String()), nil
}
type ByteArray[T ~byte] pgtype.FlatArray[T]
// Scan implements the [database/sql.Scanner] interface.
func (s *ByteArray[T]) Scan(src any) error {
var typedArray []byte
typedArray, ok := src.([]byte)
if !ok {
// tests use a different src type
err := pgtype.NewMap().SQLScanner(&typedArray).Scan(src)
if err != nil {
return err
}
}
(*s) = make(ByteArray[T], len(typedArray))
for i, value := range typedArray {
(*s)[i] = T(value)
}
return nil
}
// Value implements the [database/sql/driver.Valuer] interface.
func (s ByteArray[T]) Value() (driver.Value, error) {
if len(s) == 0 {
return nil, nil
}
typed := make([]byte, len(s))
for i, value := range s {
typed[i] = byte(value)
}
return typed, nil
}
type numberField interface {
~int8 | ~uint8 | ~int16 | ~uint16 | ~int32 | ~uint32 | ~int64 | ~uint64 | ~int | ~uint
}
type numberTypeField interface {
int8 | uint8 | int16 | uint16 | int32 | uint32 | int64 | uint64 | int | uint
}
var _ sql.Scanner = (*NumberArray[int8])(nil)
type NumberArray[F numberField] pgtype.FlatArray[F]
// Scan implements the [database/sql.Scanner] interface.
func (a *NumberArray[F]) Scan(src any) (err error) {
var (
mapper func()
scanner sql.Scanner
)
//nolint: exhaustive
// only defined types
switch reflect.TypeOf(*a).Elem().Kind() {
case reflect.Int8:
mapper, scanner = castedScan[int8](a)
case reflect.Uint8:
// we provide int16 is a workaround because pgx thinks we want to scan a byte array if we provide uint8
mapper, scanner = castedScan[int16](a)
case reflect.Int16:
mapper, scanner = castedScan[int16](a)
case reflect.Uint16:
mapper, scanner = castedScan[uint16](a)
case reflect.Int32:
mapper, scanner = castedScan[int32](a)
case reflect.Uint32:
mapper, scanner = castedScan[uint32](a)
case reflect.Int64:
mapper, scanner = castedScan[int64](a)
case reflect.Uint64:
mapper, scanner = castedScan[uint64](a)
case reflect.Int:
mapper, scanner = castedScan[int](a)
case reflect.Uint:
mapper, scanner = castedScan[uint](a)
}
if err = scanner.Scan(src); err != nil {
return err
}
mapper()
return nil
}
func castedScan[T numberTypeField, F numberField](a *NumberArray[F]) (mapper func(), scanner sql.Scanner) {
var typedArray []T
mapper = func() {
(*a) = make(NumberArray[F], len(typedArray))
for i, value := range typedArray {
(*a)[i] = F(value)
}
}
scanner = pgtype.NewMap().SQLScanner(&typedArray)
return mapper, scanner
}
type Map[V any] map[string]V
// Scan implements the [database/sql.Scanner] interface.
func (m *Map[V]) Scan(src any) error {
if src == nil {
return nil
}
bytes := src.([]byte)
if len(bytes) == 0 {
return nil
}
return json.Unmarshal(bytes, &m)
}
// Value implements the [database/sql/driver.Valuer] interface.
func (m Map[V]) Value() (driver.Value, error) {
if len(m) == 0 {
return nil, nil
}
return json.Marshal(m)
}
type Duration time.Duration
// Scan implements the [database/sql.Scanner] interface.
func (d *Duration) Scan(src any) error {
switch duration := src.(type) {
case *time.Duration:
*d = Duration(*duration)
return nil
case time.Duration:
*d = Duration(duration)
return nil
case *pgtype.Interval:
*d = intervalToDuration(duration)
return nil
case pgtype.Interval:
*d = intervalToDuration(&duration)
return nil
case int64:
*d = Duration(duration)
return nil
}
interval := new(pgtype.Interval)
if err := interval.Scan(src); err != nil {
return err
}
*d = intervalToDuration(interval)
return nil
}
func intervalToDuration(interval *pgtype.Interval) Duration {
return Duration(time.Duration(interval.Microseconds*1000) + time.Duration(interval.Days)*24*time.Hour + time.Duration(interval.Months)*30*24*time.Hour)
}
// NullDuration can be used for NULL intervals.
// If Valid is false, the scanned value was NULL
// This behavior is similar to [database/sql.NullString]
type NullDuration struct {
Valid bool
Duration time.Duration
}
// Scan implements the [database/sql.Scanner] interface.
func (d *NullDuration) Scan(src any) error {
if src == nil {
d.Duration, d.Valid = 0, false
return nil
}
duration := new(Duration)
if err := duration.Scan(src); err != nil {
return err
}
d.Duration, d.Valid = time.Duration(*duration), true
return nil
}
// JSONArray allows sending and receiving JSON arrays to and from the database.
// It implements the [database/sql.Scanner] and [database/sql/driver.Valuer] interfaces.
// Values are marshaled and unmarshaled using the [encoding/json] package.
type JSONArray[T any] []T
// NewJSONArray wraps an existing slice into a JSONArray.
func NewJSONArray[T any](a []T) JSONArray[T] {
return JSONArray[T](a)
}
// Scan implements the [database/sql.Scanner] interface.
func (a *JSONArray[T]) Scan(src any) error {
if src == nil {
*a = nil
return nil
}
bytes := src.([]byte)
if len(bytes) == 0 {
*a = nil
return nil
}
return json.Unmarshal(bytes, a)
}
// Value implements the [database/sql/driver.Valuer] interface.
func (a JSONArray[T]) Value() (driver.Value, error) {
if a == nil {
return nil, nil
}
return json.Marshal(a)
}