mirror of
https://github.com/zitadel/zitadel.git
synced 2026-11-03 12:47:09 -06:00
<!-- CURSOR_AGENT_PR_BODY_BEGIN --> - Setup 79 rewrites unique-constraint owners in one statement per file. On a large table that statement holds row locks for the whole update, and adding `LIMIT` does not help: the username statement bitmap-scans every empty `owners` value and then index-scans every user in the instance for each constraint. - The backfill joined projection tables before those tables were created and before event replay filled them. - The page cursor is a `text[]`. `database.TextArray.Value` joined elements with commas and never quoted them, so a project role key such as `a,b` became two array elements. The next page repeated the same row and the migration stopped. - Each setup 79 statement pages its projection by primary key and updates `eventstore.unique_constraints` with an equality lookup on `(instance_id, unique_type, unique_field)`. - The original field and `lower(...)` are separate `UNION` arms, so the lookup stays a primary-key probe. - Setup repeats a file until the projection page is empty. A page that matches no constraint still advances the cursor. - `BatchSize` is a `uint16`, configured like `ForceFinalize` (`cmd/setup/steps.yaml`, `ZITADEL_BACKFILLUNIQUECONSTRAINTOWNERS_BATCHSIZE`) and is not stored on lastRun. The default in `steps.yaml` is 5000. - Each page logs the file, the page index, and the number of rows updated. - The backfill runs after projection prefill. When `InitProjections` is disabled or setup is for mirror, it still runs against the projection rows already stored. - `TextArray.Value` quotes every element and backslash-escapes `"` and `\`. An empty array is still SQL `NULL`, so existing `ANY($1)` callers stay unchanged. The setup statements already cast `$1` to `text[]`. - Integration tests for the username and org-domain statements pass an empty cursor and a page size. - `TestTextArray_Value` expects quoted elements, and a round trip through `Scan` covers a comma, a quote, and a backslash. - initial fix https://github.com/zitadel/zitadel/pull/12740 - fix of the setup job, broken by the first fix https://github.com/zitadel/zitadel/pull/12785 - needs to be a `fix` tag because `perf` wouldn't bump the version - Based on `owner-app-fix`. - A setup process already running the old statement keeps that plan until it is restarted. <!-- CURSOR_AGENT_PR_BODY_END --> <div><a href="https://cursor.com/agents/bc-31bfb5c4-5888-43f0-85cf-af1af097ca30?cursor_ref=pr_footer&cursor_cta=open_in_web"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/open-in-web-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/open-in-web-light.png"><img alt="Open in Web" width="114" height="28" src="https://cursor.com/assets/images/open-in-web-dark.png"></picture></a> <a href="https://cursor.com/background-agent?bcId=bc-31bfb5c4-5888-43f0-85cf-af1af097ca30&cursor_ref=pr_footer&cursor_cta=open_in_cursor"><picture><source media="(prefers-color-scheme: dark)" srcset="https://cursor.com/assets/images/open-in-cursor-dark.png"><source media="(prefers-color-scheme: light)" srcset="https://cursor.com/assets/images/open-in-cursor-light.png"><img alt="Open in Cursor" width="131" height="28" src="https://cursor.com/assets/images/open-in-cursor-dark.png"></picture></a> </div> --------- Co-authored-by: Cursor Agent <cursoragent@cursor.com> Co-authored-by: Silvan <adlerhurst@users.noreply.github.com>
274 lines
6.2 KiB
Go
274 lines
6.2 KiB
Go
package database
|
|
|
|
import (
|
|
"database/sql"
|
|
"database/sql/driver"
|
|
"encoding/json"
|
|
"reflect"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/jackc/pgx/v5/pgtype"
|
|
)
|
|
|
|
type TextArray[T ~string] pgtype.FlatArray[T]
|
|
|
|
// Scan implements the [database/sql.Scanner] interface.
|
|
func (s *TextArray[T]) Scan(src any) error {
|
|
var typedArray []string
|
|
err := pgtype.NewMap().SQLScanner(&typedArray).Scan(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
(*s) = make(TextArray[T], len(typedArray))
|
|
for i, value := range typedArray {
|
|
(*s)[i] = T(value)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Value implements the [database/sql/driver.Valuer] interface.
|
|
// Every element is quoted so commas, quotes, and backslashes stay inside one element.
|
|
// An empty array is NULL, which keeps ANY($1) filters unchanged.
|
|
func (s TextArray[T]) Value() (driver.Value, error) {
|
|
if len(s) == 0 {
|
|
return nil, nil
|
|
}
|
|
|
|
var b strings.Builder
|
|
b.WriteByte('{')
|
|
for i, value := range s {
|
|
if i > 0 {
|
|
b.WriteByte(',')
|
|
}
|
|
b.WriteByte('"')
|
|
for _, r := range string(value) {
|
|
if r == '"' || r == '\\' {
|
|
b.WriteByte('\\')
|
|
}
|
|
b.WriteRune(r)
|
|
}
|
|
b.WriteByte('"')
|
|
}
|
|
b.WriteByte('}')
|
|
return []byte(b.String()), nil
|
|
}
|
|
|
|
type ByteArray[T ~byte] pgtype.FlatArray[T]
|
|
|
|
// Scan implements the [database/sql.Scanner] interface.
|
|
func (s *ByteArray[T]) Scan(src any) error {
|
|
var typedArray []byte
|
|
typedArray, ok := src.([]byte)
|
|
if !ok {
|
|
// tests use a different src type
|
|
err := pgtype.NewMap().SQLScanner(&typedArray).Scan(src)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
(*s) = make(ByteArray[T], len(typedArray))
|
|
for i, value := range typedArray {
|
|
(*s)[i] = T(value)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Value implements the [database/sql/driver.Valuer] interface.
|
|
func (s ByteArray[T]) Value() (driver.Value, error) {
|
|
if len(s) == 0 {
|
|
return nil, nil
|
|
}
|
|
typed := make([]byte, len(s))
|
|
|
|
for i, value := range s {
|
|
typed[i] = byte(value)
|
|
}
|
|
|
|
return typed, nil
|
|
}
|
|
|
|
type numberField interface {
|
|
~int8 | ~uint8 | ~int16 | ~uint16 | ~int32 | ~uint32 | ~int64 | ~uint64 | ~int | ~uint
|
|
}
|
|
|
|
type numberTypeField interface {
|
|
int8 | uint8 | int16 | uint16 | int32 | uint32 | int64 | uint64 | int | uint
|
|
}
|
|
|
|
var _ sql.Scanner = (*NumberArray[int8])(nil)
|
|
|
|
type NumberArray[F numberField] pgtype.FlatArray[F]
|
|
|
|
// Scan implements the [database/sql.Scanner] interface.
|
|
func (a *NumberArray[F]) Scan(src any) (err error) {
|
|
var (
|
|
mapper func()
|
|
scanner sql.Scanner
|
|
)
|
|
|
|
//nolint: exhaustive
|
|
// only defined types
|
|
switch reflect.TypeOf(*a).Elem().Kind() {
|
|
case reflect.Int8:
|
|
mapper, scanner = castedScan[int8](a)
|
|
case reflect.Uint8:
|
|
// we provide int16 is a workaround because pgx thinks we want to scan a byte array if we provide uint8
|
|
mapper, scanner = castedScan[int16](a)
|
|
case reflect.Int16:
|
|
mapper, scanner = castedScan[int16](a)
|
|
case reflect.Uint16:
|
|
mapper, scanner = castedScan[uint16](a)
|
|
case reflect.Int32:
|
|
mapper, scanner = castedScan[int32](a)
|
|
case reflect.Uint32:
|
|
mapper, scanner = castedScan[uint32](a)
|
|
case reflect.Int64:
|
|
mapper, scanner = castedScan[int64](a)
|
|
case reflect.Uint64:
|
|
mapper, scanner = castedScan[uint64](a)
|
|
case reflect.Int:
|
|
mapper, scanner = castedScan[int](a)
|
|
case reflect.Uint:
|
|
mapper, scanner = castedScan[uint](a)
|
|
}
|
|
|
|
if err = scanner.Scan(src); err != nil {
|
|
return err
|
|
}
|
|
mapper()
|
|
|
|
return nil
|
|
}
|
|
|
|
func castedScan[T numberTypeField, F numberField](a *NumberArray[F]) (mapper func(), scanner sql.Scanner) {
|
|
var typedArray []T
|
|
|
|
mapper = func() {
|
|
(*a) = make(NumberArray[F], len(typedArray))
|
|
for i, value := range typedArray {
|
|
(*a)[i] = F(value)
|
|
}
|
|
}
|
|
scanner = pgtype.NewMap().SQLScanner(&typedArray)
|
|
|
|
return mapper, scanner
|
|
}
|
|
|
|
type Map[V any] map[string]V
|
|
|
|
// Scan implements the [database/sql.Scanner] interface.
|
|
func (m *Map[V]) Scan(src any) error {
|
|
if src == nil {
|
|
return nil
|
|
}
|
|
|
|
bytes := src.([]byte)
|
|
if len(bytes) == 0 {
|
|
return nil
|
|
}
|
|
|
|
return json.Unmarshal(bytes, &m)
|
|
}
|
|
|
|
// Value implements the [database/sql/driver.Valuer] interface.
|
|
func (m Map[V]) Value() (driver.Value, error) {
|
|
if len(m) == 0 {
|
|
return nil, nil
|
|
}
|
|
return json.Marshal(m)
|
|
}
|
|
|
|
type Duration time.Duration
|
|
|
|
// Scan implements the [database/sql.Scanner] interface.
|
|
func (d *Duration) Scan(src any) error {
|
|
switch duration := src.(type) {
|
|
case *time.Duration:
|
|
*d = Duration(*duration)
|
|
return nil
|
|
case time.Duration:
|
|
*d = Duration(duration)
|
|
return nil
|
|
case *pgtype.Interval:
|
|
*d = intervalToDuration(duration)
|
|
return nil
|
|
case pgtype.Interval:
|
|
*d = intervalToDuration(&duration)
|
|
return nil
|
|
case int64:
|
|
*d = Duration(duration)
|
|
return nil
|
|
}
|
|
interval := new(pgtype.Interval)
|
|
if err := interval.Scan(src); err != nil {
|
|
return err
|
|
}
|
|
*d = intervalToDuration(interval)
|
|
return nil
|
|
}
|
|
|
|
func intervalToDuration(interval *pgtype.Interval) Duration {
|
|
return Duration(time.Duration(interval.Microseconds*1000) + time.Duration(interval.Days)*24*time.Hour + time.Duration(interval.Months)*30*24*time.Hour)
|
|
}
|
|
|
|
// NullDuration can be used for NULL intervals.
|
|
// If Valid is false, the scanned value was NULL
|
|
// This behavior is similar to [database/sql.NullString]
|
|
type NullDuration struct {
|
|
Valid bool
|
|
Duration time.Duration
|
|
}
|
|
|
|
// Scan implements the [database/sql.Scanner] interface.
|
|
func (d *NullDuration) Scan(src any) error {
|
|
if src == nil {
|
|
d.Duration, d.Valid = 0, false
|
|
return nil
|
|
}
|
|
duration := new(Duration)
|
|
if err := duration.Scan(src); err != nil {
|
|
return err
|
|
}
|
|
d.Duration, d.Valid = time.Duration(*duration), true
|
|
return nil
|
|
}
|
|
|
|
// JSONArray allows sending and receiving JSON arrays to and from the database.
|
|
// It implements the [database/sql.Scanner] and [database/sql/driver.Valuer] interfaces.
|
|
// Values are marshaled and unmarshaled using the [encoding/json] package.
|
|
type JSONArray[T any] []T
|
|
|
|
// NewJSONArray wraps an existing slice into a JSONArray.
|
|
func NewJSONArray[T any](a []T) JSONArray[T] {
|
|
return JSONArray[T](a)
|
|
}
|
|
|
|
// Scan implements the [database/sql.Scanner] interface.
|
|
func (a *JSONArray[T]) Scan(src any) error {
|
|
if src == nil {
|
|
*a = nil
|
|
return nil
|
|
}
|
|
|
|
bytes := src.([]byte)
|
|
if len(bytes) == 0 {
|
|
*a = nil
|
|
return nil
|
|
}
|
|
|
|
return json.Unmarshal(bytes, a)
|
|
}
|
|
|
|
// Value implements the [database/sql/driver.Valuer] interface.
|
|
func (a JSONArray[T]) Value() (driver.Value, error) {
|
|
if a == nil {
|
|
return nil, nil
|
|
}
|
|
return json.Marshal(a)
|
|
}
|