IntenseWebs/gitea
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix postgres ID sequences broken by recreate-table (#15015) (#15029)

Browse Source 
Backport #15015

Unfortunately there is a subtle problem with recreatetable on postgres which
leads to the sequences not being renamed and not being left at 0.

Fix #14725

Signed-off-by: Andrew Thornton <art27@cantab.net>
This commit is contained in:
zeripath 2021-03-19 03:23:58 +00:00 committed by GitHub
parent 645c0d8abd
commit 909f2be99d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 140 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
cmd/doctor.go
View File

@ -670,6 +670,23 @@ func runDoctorCheckDBConsistency(ctx *cli.Context) ([]string, error) {
}
}
if setting.Database.UsePostgreSQL {
count, err = models.CountBadSequences()
if err != nil {
return nil, err
}
if count > 0 {
if ctx.Bool("fix") {
err := models.FixBadSequences()
if err != nil {
return nil, err
}
results = append(results, fmt.Sprintf("%d sequences updated", count))
} else {
results = append(results, fmt.Sprintf("%d sequences with incorrect values", count))
}
}
}
//ToDo: function to recalc all counters
return results, nil

61
models/consistency.go
View File

@ -5,10 +5,13 @@
package models
import (
"fmt"
"reflect"
"regexp"
"strings"
"testing"
"code.gitea.io/gitea/modules/setting"
"github.com/stretchr/testify/assert"
"xorm.io/builder"
)
@ -295,3 +298,61 @@ func FixNullArchivedRepository() (int64, error) {
IsArchived: false,
})
}
// CountBadSequences looks for broken sequences from recreate-table mistakes
func CountBadSequences() (int64, error) {
if !setting.Database.UsePostgreSQL {
return 0, nil
}
sess := x.NewSession()
defer sess.Close()
var sequences []string
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
return 0, err
}
sess.Engine().SetSchema(schema)
return int64(len(sequences)), nil
}
// FixBadSequences fixes for broken sequences from recreate-table mistakes
func FixBadSequences() error {
if !setting.Database.UsePostgreSQL {
return nil
}
sess := x.NewSession()
defer sess.Close()
if err := sess.Begin(); err != nil {
return err
}
var sequences []string
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__%_id_seq%' AND sequence_catalog = ?", setting.Database.Name).Find(&sequences); err != nil {
return err
}
sess.Engine().SetSchema(schema)
sequenceRegexp := regexp.MustCompile(`tmp_recreate__(\w+)_id_seq.*`)
for _, sequence := range sequences {
tableName := sequenceRegexp.FindStringSubmatch(sequence)[1]
newSequenceName := tableName + "_id_seq"
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
return err
}
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
return err
}
}
return sess.Commit()
}

63
models/migrations/migrations.go
View File

@ -516,6 +516,31 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
return err
}
case setting.Database.UsePostgreSQL:
var originalSequences []string
type sequenceData struct {
LastValue int `xorm:"'last_value'"`
IsCalled bool `xorm:"'is_called'"`
}
sequenceMap := map[string]sequenceData{}
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&originalSequences); err != nil {
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
return err
}
sess.Engine().SetSchema(schema)
for _, sequence := range originalSequences {
sequenceData := sequenceData{}
if _, err := sess.Table(sequence).Cols("last_value", "is_called").Get(&sequenceData); err != nil {
log.Error("Unable to get last_value and is_called from %s. Error: %v", sequence, err)
return err
}
sequenceMap[sequence] = sequenceData
}
// CASCADE causes postgres to drop all the constraints on the old table
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s` CASCADE", tableName)); err != nil {
log.Error("Unable to drop old table %s. Error: %v", tableName, err)
@ -529,7 +554,6 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
}
var indices []string
schema := sess.Engine().Dialect().URI().Schema
sess.Engine().SetSchema("")
if err := sess.Table("pg_indexes").Cols("indexname").Where("tablename = ? ", tableName).Find(&indices); err != nil {
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
@ -545,6 +569,43 @@ func recreateTable(sess *xorm.Session, bean interface{}) error {
}
}
var sequences []string
sess.Engine().SetSchema("")
if err := sess.Table("information_schema.sequences").Cols("sequence_name").Where("sequence_name LIKE 'tmp_recreate__' || ? || '_%' AND sequence_catalog = ?", tableName, setting.Database.Name).Find(&sequences); err != nil {
log.Error("Unable to rename %s to %s. Error: %v", tempTableName, tableName, err)
return err
}
sess.Engine().SetSchema(schema)
for _, sequence := range sequences {
newSequenceName := strings.Replace(sequence, "tmp_recreate__", "", 1)
if _, err := sess.Exec(fmt.Sprintf("ALTER SEQUENCE `%s` RENAME TO `%s`", sequence, newSequenceName)); err != nil {
log.Error("Unable to rename %s sequence to %s. Error: %v", sequence, newSequenceName, err)
return err
}
val, ok := sequenceMap[newSequenceName]
if newSequenceName == tableName+"_id_seq" {
if ok && val.LastValue != 0 {
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
return err
}
} else {
// We're going to try to guess this
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', COALESCE((SELECT MAX(id)+1 FROM `%s`), 1), false)", newSequenceName, tableName)); err != nil {
log.Error("Unable to reset %s. Error: %v", newSequenceName, err)
return err
}
}
} else if ok {
if _, err := sess.Exec(fmt.Sprintf("SELECT setval('%s', %d, %t)", newSequenceName, val.LastValue, val.IsCalled)); err != nil {
log.Error("Unable to reset %s to %d. Error: %v", newSequenceName, val, err)
return err
}
}
}
case setting.Database.UseMSSQL:
// MSSQL will drop all the constraints on the old table
if _, err := sess.Exec(fmt.Sprintf("DROP TABLE `%s`", tableName)); err != nil {