IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

Remote any references to sqlite (#16980)

Browse Source 
* Remote any references to sqlite

* Fix specs
This commit is contained in:
John Tzikas
2021-02-19 11:09:52 +02:00
committed by GitHub
parent 7b8cff1962
commit 8d6dd195e1
3 changed files with 119 additions and 147 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
store/sqlstore/store.go
View File

@@ -73,10 +73,6 @@ const (
ExitRemoveIndexMySQL = 122
ExitRemoveIndexMissing = 123
ExitRemoveTable = 134
ExitCreateIndexSqlite = 135
ExitRemoveIndexSqlite = 136
ExitTableExists_SQLITE = 137
ExitDoesColumnExistsSqlite = 138
ExitAlterPrimaryKey = 139
)
@@ -274,9 +270,7 @@ func setupConnection(con_type string, dataSource string, settings *model.SqlSett
connectionTimeout := time.Duration(*settings.QueryTimeout) * time.Second
if *settings.DriverName == model.DATABASE_DRIVER_SQLITE {
dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.SqliteDialect{}, QueryTimeout: connectionTimeout}
} else if *settings.DriverName == model.DATABASE_DRIVER_MYSQL {
if *settings.DriverName == model.DATABASE_DRIVER_MYSQL {
dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.MySQLDialect{Engine: "InnoDB", Encoding: "UTF8MB4"}, QueryTimeout: connectionTimeout}
} else if *settings.DriverName == model.DATABASE_DRIVER_POSTGRES {
dbmap = &gorp.DbMap{Db: db, TypeConverter: mattermConverter{}, Dialect: gorp.PostgresDialect{}, QueryTimeout: connectionTimeout}
@@ -341,8 +335,6 @@ func (ss *SqlStore) GetDbVersion(numerical bool) (string, error) {
}
} else if ss.DriverName() == model.DATABASE_DRIVER_MYSQL {
sqlVersion = `SELECT version()`
} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
sqlVersion = `SELECT sqlite_version()`
} else {
return "", errors.New("Not supported driver")
}
@@ -468,20 +460,6 @@ func (ss *SqlStore) DoesTableExist(tableName string) bool {
return count > 0
} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
count, err := ss.GetMaster().SelectInt(
`SELECT count(name) FROM sqlite_master WHERE type='table' AND name=?`,
tableName,
)
if err != nil {
mlog.Critical("Failed to check if table exists", mlog.Err(err))
time.Sleep(time.Second)
os.Exit(ExitTableExists_SQLITE)
}
return count > 0
} else {
mlog.Critical("Failed to check if column exists because of missing driver")
time.Sleep(time.Second)
@@ -537,21 +515,6 @@ func (ss *SqlStore) DoesColumnExist(tableName string, columnName string) bool {
return count > 0
} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
count, err := ss.GetMaster().SelectInt(
`SELECT COUNT(*) FROM pragma_table_info(?) WHERE name=?`,
tableName,
columnName,
)
if err != nil {
mlog.Critical("Failed to check if column exists", mlog.Err(err))
time.Sleep(time.Second)
os.Exit(ExitDoesColumnExistsSqlite)
}
return count > 0
} else {
mlog.Critical("Failed to check if column exists because of missing driver")
time.Sleep(time.Second)
@@ -791,10 +754,6 @@ func (ss *SqlStore) AlterColumnDefaultIfExists(tableName string, columnName stri
tableName = strings.ToLower(tableName)
columnName = strings.ToLower(columnName)
defaultValue = *postgresColDefault
} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
// SQLite doesn't support altering column defaults, but we don't use this in
// production so just ignore.
return true
} else {
mlog.Critical("Failed to alter column default because of missing driver")
time.Sleep(time.Second)
@@ -850,9 +809,6 @@ func (ss *SqlStore) AlterPrimaryKey(tableName string, columnNames []string) bool
c.contype = 'p'
AND c.conrelid = '` + strings.ToLower(tableName) + `'::REGCLASS`
currentPrimaryKey, err = ss.GetMaster().SelectStr(query)
case model.DATABASE_DRIVER_SQLITE:
// SQLite doesn't support altering primary key
return true
}
if err != nil {
mlog.Critical("Failed to get current primary key", mlog.String("table", tableName), mlog.Err(err))
@@ -957,13 +913,6 @@ func (ss *SqlStore) createIndexIfNotExists(indexName string, tableName string, c
time.Sleep(time.Second)
os.Exit(ExitCreateIndexFullMySQL)
}
} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
_, err := ss.GetMaster().ExecNoTimeout("CREATE INDEX IF NOT EXISTS " + indexName + " ON " + tableName + " (" + strings.Join(columnNames, ", ") + ")")
if err != nil {
mlog.Critical("Failed to create index", mlog.Err(err))
time.Sleep(time.Second)
os.Exit(ExitCreateIndexSqlite)
}
} else {
mlog.Critical("Failed to create index because of missing driver")
time.Sleep(time.Second)
@@ -1009,13 +958,6 @@ func (ss *SqlStore) RemoveIndexIfExists(indexName string, tableName string) bool
time.Sleep(time.Second)
os.Exit(ExitRemoveIndexMySQL)
}
} else if ss.DriverName() == model.DATABASE_DRIVER_SQLITE {
_, err := ss.GetMaster().ExecNoTimeout("DROP INDEX IF EXISTS " + indexName)
if err != nil {
mlog.Critical("Failed to remove index", mlog.Err(err))
time.Sleep(time.Second)
os.Exit(ExitRemoveIndexSqlite)
}
} else {
mlog.Critical("Failed to create index because of missing driver")
time.Sleep(time.Second)

205
store/sqlstore/store_test.go
View File

@@ -162,10 +162,15 @@ func tearDownStores() {
// before the fix in MM-28397.
// Keeping it here to help avoiding future regressions.
func TestStoreLicenseRace(t *testing.T) {
settings := makeSqlSettings(model.DATABASE_DRIVER_SQLITE)
settings.DataSourceReplicas = []string{":memory:"}
settings.DataSourceSearchReplicas = []string{":memory:"}
settings := makeSqlSettings(model.DATABASE_DRIVER_POSTGRES)
store := New(*settings, nil)
defer func() {
store.Close()
for _, replica := range store.searchReplicas {
require.NoError(t, replica.Db.Close())
}
storetest.CleanupSqlSettings(settings)
}()
wg := sync.WaitGroup{}
wg.Add(3)
@@ -191,54 +196,54 @@ func TestStoreLicenseRace(t *testing.T) {
func TestGetReplica(t *testing.T) {
t.Parallel()
testCases := []struct {
Description string
DataSourceReplicas []string
DataSourceSearchReplicas []string
Description string
DataSourceReplicaNum int
DataSourceSearchReplicaNum int
}{
{
"no replicas",
[]string{},
[]string{},
0,
0,
},
{
"one source replica",
[]string{":memory:"},
[]string{},
1,
0,
},
{
"multiple source replicas",
[]string{":memory:", ":memory:", ":memory:"},
[]string{},
3,
0,
},
{
"one source search replica",
[]string{},
[]string{":memory:"},
0,
1,
},
{
"multiple source search replicas",
[]string{},
[]string{":memory:", ":memory:", ":memory:"},
0,
3,
},
{
"one source replica, one source search replica",
[]string{":memory:"},
[]string{":memory:"},
1,
1,
},
{
"one source replica, multiple source search replicas",
[]string{":memory:"},
[]string{":memory:", ":memory:", ":memory:"},
1,
3,
},
{
"multiple source replica, one source search replica",
[]string{":memory:", ":memory:", ":memory:"},
[]string{":memory:"},
3,
1,
},
{
"multiple source replica, multiple source search replicas",
[]string{":memory:", ":memory:", ":memory:"},
[]string{":memory:", ":memory:", ":memory:"},
3,
3,
},
}
@@ -247,10 +252,27 @@ func TestGetReplica(t *testing.T) {
t.Run(testCase.Description+" with license", func(t *testing.T) {
t.Parallel()
settings := makeSqlSettings(model.DATABASE_DRIVER_SQLITE)
settings.DataSourceReplicas = testCase.DataSourceReplicas
settings.DataSourceSearchReplicas = testCase.DataSourceSearchReplicas
settings := makeSqlSettings(model.DATABASE_DRIVER_POSTGRES)
dataSourceReplicas := []string{}
dataSourceSearchReplicas := []string{}
for i := 0; i < testCase.DataSourceReplicaNum; i++ {
dataSourceReplicas = append(dataSourceReplicas, *settings.DataSource)
}
for i := 0; i < testCase.DataSourceSearchReplicaNum; i++ {
dataSourceSearchReplicas = append(dataSourceSearchReplicas, *settings.DataSource)
}
settings.DataSourceReplicas = dataSourceReplicas
settings.DataSourceSearchReplicas = dataSourceSearchReplicas
store := New(*settings, nil)
defer func() {
store.Close()
for _, replica := range store.searchReplicas {
require.NoError(t, replica.Db.Close())
}
storetest.CleanupSqlSettings(settings)
}()
store.UpdateLicense(&model.License{})
replicas := make(map[*gorp.DbMap]bool)
@@ -263,9 +285,9 @@ func TestGetReplica(t *testing.T) {
searchReplicas[store.GetSearchReplica()] = true
}
if len(testCase.DataSourceReplicas) > 0 {
if testCase.DataSourceReplicaNum > 0 {
// If replicas were defined, ensure none are the master.
assert.Len(t, replicas, len(testCase.DataSourceReplicas))
assert.Len(t, replicas, testCase.DataSourceReplicaNum)
for replica := range replicas {
assert.NotSame(t, store.GetMaster(), replica)
@@ -278,9 +300,9 @@ func TestGetReplica(t *testing.T) {
}
}
if len(testCase.DataSourceSearchReplicas) > 0 {
if testCase.DataSourceSearchReplicaNum > 0 {
// If search replicas were defined, ensure none are the master nor the replicas.
assert.Len(t, searchReplicas, len(testCase.DataSourceSearchReplicas))
assert.Len(t, searchReplicas, testCase.DataSourceSearchReplicaNum)
for searchReplica := range searchReplicas {
assert.NotSame(t, store.GetMaster(), searchReplica)
@@ -288,12 +310,12 @@ func TestGetReplica(t *testing.T) {
assert.NotSame(t, searchReplica, replica)
}
}
} else if len(testCase.DataSourceReplicas) > 0 {
} else if testCase.DataSourceReplicaNum > 0 {
assert.Equal(t, len(replicas), len(searchReplicas))
for k := range replicas {
assert.True(t, searchReplicas[k])
}
} else if len(testCase.DataSourceReplicas) == 0 && assert.Len(t, searchReplicas, 1) {
} else if testCase.DataSourceReplicaNum == 0 && assert.Len(t, searchReplicas, 1) {
// Otherwise ensure the search replicas contains the master.
for searchReplica := range searchReplicas {
assert.Same(t, store.GetMaster(), searchReplica)
@@ -304,10 +326,26 @@ func TestGetReplica(t *testing.T) {
t.Run(testCase.Description+" without license", func(t *testing.T) {
t.Parallel()
settings := makeSqlSettings(model.DATABASE_DRIVER_SQLITE)
settings.DataSourceReplicas = testCase.DataSourceReplicas
settings.DataSourceSearchReplicas = testCase.DataSourceSearchReplicas
settings := makeSqlSettings(model.DATABASE_DRIVER_POSTGRES)
dataSourceReplicas := []string{}
dataSourceSearchReplicas := []string{}
for i := 0; i < testCase.DataSourceReplicaNum; i++ {
dataSourceReplicas = append(dataSourceReplicas, *settings.DataSource)
}
for i := 0; i < testCase.DataSourceSearchReplicaNum; i++ {
dataSourceSearchReplicas = append(dataSourceSearchReplicas, *settings.DataSource)
}
settings.DataSourceReplicas = dataSourceReplicas
settings.DataSourceSearchReplicas = dataSourceSearchReplicas
store := New(*settings, nil)
defer func() {
store.Close()
for _, replica := range store.searchReplicas {
require.NoError(t, replica.Db.Close())
}
storetest.CleanupSqlSettings(settings)
}()
replicas := make(map[*gorp.DbMap]bool)
for i := 0; i < 5; i++ {
@@ -319,7 +357,7 @@ func TestGetReplica(t *testing.T) {
searchReplicas[store.GetSearchReplica()] = true
}
if len(testCase.DataSourceReplicas) > 0 {
if testCase.DataSourceReplicaNum > 0 {
// If replicas were defined, ensure none are the master.
assert.Len(t, replicas, 1)
@@ -334,7 +372,7 @@ func TestGetReplica(t *testing.T) {
}
}
if len(testCase.DataSourceSearchReplicas) > 0 {
if testCase.DataSourceSearchReplicaNum > 0 {
// If search replicas were defined, ensure none are the master nor the replicas.
assert.Len(t, searchReplicas, 1)
@@ -342,7 +380,7 @@ func TestGetReplica(t *testing.T) {
assert.Same(t, store.GetMaster(), searchReplica)
}
} else if len(testCase.DataSourceReplicas) > 0 {
} else if testCase.DataSourceReplicaNum > 0 {
assert.Equal(t, len(replicas), len(searchReplicas))
for k := range replicas {
assert.True(t, searchReplicas[k])
@@ -361,7 +399,6 @@ func TestGetDbVersion(t *testing.T) {
testDrivers := []string{
model.DATABASE_DRIVER_POSTGRES,
model.DATABASE_DRIVER_MYSQL,
model.DATABASE_DRIVER_SQLITE,
}
for _, driver := range testDrivers {
@@ -380,63 +417,63 @@ func TestGetDbVersion(t *testing.T) {
func TestGetAllConns(t *testing.T) {
t.Parallel()
testCases := []struct {
Description string
DataSourceReplicas []string
DataSourceSearchReplicas []string
ExpectedNumConnections int
Description string
DataSourceReplicaNum int
DataSourceSearchReplicaNum int
ExpectedNumConnections int
}{
{
"no replicas",
[]string{},
[]string{},
0,
0,
1,
},
{
"one source replica",
[]string{":memory:"},
[]string{},
1,
0,
2,
},
{
"multiple source replicas",
[]string{":memory:", ":memory:", ":memory:"},
[]string{},
3,
0,
4,
},
{
"one source search replica",
[]string{},
[]string{":memory:"},
0,
1,
1,
},
{
"multiple source search replicas",
[]string{},
[]string{":memory:", ":memory:", ":memory:"},
0,
3,
1,
},
{
"one source replica, one source search replica",
[]string{":memory:"},
[]string{":memory:"},
1,
1,
2,
},
{
"one source replica, multiple source search replicas",
[]string{":memory:"},
[]string{":memory:", ":memory:", ":memory:"},
1,
3,
2,
},
{
"multiple source replica, one source search replica",
[]string{":memory:", ":memory:", ":memory:"},
[]string{":memory:"},
3,
1,
4,
},
{
"multiple source replica, multiple source search replicas",
[]string{":memory:", ":memory:", ":memory:"},
[]string{":memory:", ":memory:", ":memory:"},
3,
3,
4,
},
}
@@ -445,10 +482,26 @@ func TestGetAllConns(t *testing.T) {
testCase := testCase
t.Run(testCase.Description, func(t *testing.T) {
t.Parallel()
settings := makeSqlSettings(model.DATABASE_DRIVER_SQLITE)
settings.DataSourceReplicas = testCase.DataSourceReplicas
settings.DataSourceSearchReplicas = testCase.DataSourceSearchReplicas
settings := makeSqlSettings(model.DATABASE_DRIVER_POSTGRES)
dataSourceReplicas := []string{}
dataSourceSearchReplicas := []string{}
for i := 0; i < testCase.DataSourceReplicaNum; i++ {
dataSourceReplicas = append(dataSourceReplicas, *settings.DataSource)
}
for i := 0; i < testCase.DataSourceSearchReplicaNum; i++ {
dataSourceSearchReplicas = append(dataSourceSearchReplicas, *settings.DataSource)
}
settings.DataSourceReplicas = dataSourceReplicas
settings.DataSourceSearchReplicas = dataSourceSearchReplicas
store := New(*settings, nil)
defer func() {
store.Close()
for _, replica := range store.searchReplicas {
require.NoError(t, replica.Db.Close())
}
storetest.CleanupSqlSettings(settings)
}()
assert.Len(t, store.GetAllConns(), testCase.ExpectedNumConnections)
})
@@ -503,29 +556,7 @@ func makeSqlSettings(driver string) *model.SqlSettings {
return storetest.MakeSqlSettings(driver)
case model.DATABASE_DRIVER_MYSQL:
return storetest.MakeSqlSettings(driver)
case model.DATABASE_DRIVER_SQLITE:
return makeSqliteSettings()
}
return nil
}
func makeSqliteSettings() *model.SqlSettings {
driverName := model.DATABASE_DRIVER_SQLITE
dataSource := ":memory:"
maxIdleConns := 1
connMaxLifetimeMilliseconds := 3600000
connMaxIdleTimeMilliseconds := 300000
maxOpenConns := 1
queryTimeout := 5
return &model.SqlSettings{
DriverName: &driverName,
DataSource: &dataSource,
MaxIdleConns: &maxIdleConns,
ConnMaxLifetimeMilliseconds: &connMaxLifetimeMilliseconds,
ConnMaxIdleTimeMilliseconds: &connMaxIdleTimeMilliseconds,
MaxOpenConns: &maxOpenConns,
QueryTimeout: &queryTimeout,
}
}