[MM-31094] Replication Lag (#16888)

* MM-31094: Adds tooling to develop and test using a MySQL instance with replication lag. Adds some lazy lookups to fallback to master if results are not found.

* MM-31094: Removes mysql-read-replica from default docker services.

* MM-31094: Switches (store..SessionStore).Get and (store.TeamStore).GetMember to using context.Context.

* MM-31094: Updates (store.UsersStore).Get to use context.

* MM-31094: Updates (store.PostStore).Get to use context.

* MM-31094: Removes feature flag and config setting.

* MM-31094: Rolls back some master reads.

* MM-31094: Rolls a non-cache read.

* MM-31094: Removes feature flag from the store.

* MM-31094: Removes unused constant and struct field.

* MM-31094: Removes some old feature flag references.

* MM-31094: Fixes some tests.

* MM-31094: App layers fix.

* MM-31094: Fixes mocks.

* MM-31094: Don't reparse flag.

* MM-31094: No reparse.

* MM-31094: Removed unused FeatureFlags field.

* MM-31094: Removes unnecessary feature flags variable declarations.

* MM-31094: Fixes copy-paste error.

* MM-31094: Fixes logical error.

* MM-30194: Removes test method from store.

* Revert "MM-30194: Removes test method from store."

This reverts commit d5a6e8529b.

* MM-31094: Conforming to make's strange syntax.

* MM-31094: Configures helper for read replica with option.

* MM-31094: Adds some missing ctx's.

* MM-31094: WIP

* MM-31094: Updates test names.

* MM-31094: WIP

* MM-31094: Removes unnecessary master reads.

* MM-31094: ID case changes out of scope.

* MM-31094: Removes unused context.

* MM-31094: Switches to a helper. Removes some var naming changes. Fixes a merge error.

* MM-31094: Removes SQLITE db driver ref.

* MM-31094: Layer generate fix.

* MM-31094: Removes unnecessary changes.

* MM-31094: Moves test method.

* MM-31094: Re-add previous fix.

* MM-31094: Removes make command for dev.

* MM-31094: Fix for login.

Co-authored-by: Mattermod <mattermod@users.noreply.github.com>

testlib/helper.go

@@ -33,11 +33,13 @@ type MainHelper struct {
 
 	status           int
 	testResourcePath string
+	replicas         []string
 }
 
 type HelperOptions struct {
 	EnableStore     bool
 	EnableResources bool
+	WithReadReplica bool
 }
 
 func NewMainHelper() *MainHelper {
@@ -65,7 +67,7 @@ func NewMainHelperWithOptions(options *HelperOptions) *MainHelper {
 
 	if options != nil {
 		if options.EnableStore && !testing.Short() {
-			mainHelper.setupStore()
+			mainHelper.setupStore(options.WithReadReplica)
 		}
 
 		if options.EnableResources {
@@ -99,13 +101,14 @@ func (h *MainHelper) Main(m *testing.M) {
 	h.status = m.Run()
 }
 
-func (h *MainHelper) setupStore() {
+func (h *MainHelper) setupStore(withReadReplica bool) {
 	driverName := os.Getenv("MM_SQLSETTINGS_DRIVERNAME")
 	if driverName == "" {
 		driverName = model.DATABASE_DRIVER_POSTGRES
 	}
 
-	h.Settings = storetest.MakeSqlSettings(driverName)
+	h.Settings = storetest.MakeSqlSettings(driverName, withReadReplica)
+	h.replicas = h.Settings.DataSourceReplicas
 
 	config := &model.Config{}
 	config.SetDefaults()
@@ -118,6 +121,26 @@ func (h *MainHelper) setupStore() {
 	}, h.SearchEngine, config)
 }
 
+func (h *MainHelper) ToggleReplicasOff() {
+	if h.SQLStore.GetLicense() == nil {
+		panic("expecting a license to use this")
+	}
+	h.Settings.DataSourceReplicas = []string{}
+	lic := h.SQLStore.GetLicense()
+	h.SQLStore = sqlstore.New(*h.Settings, nil)
+	h.SQLStore.UpdateLicense(lic)
+}
+
+func (h *MainHelper) ToggleReplicasOn() {
+	if h.SQLStore.GetLicense() == nil {
+		panic("expecting a license to use this")
+	}
+	h.Settings.DataSourceReplicas = h.replicas
+	lic := h.SQLStore.GetLicense()
+	h.SQLStore = sqlstore.New(*h.Settings, nil)
+	h.SQLStore.UpdateLicense(lic)
+}
+
 func (h *MainHelper) setupResources() {
 	var err error
 	h.testResourcePath, err = SetupTestResources()
@@ -234,3 +257,36 @@ func (h *MainHelper) GetSearchEngine() *searchengine.Broker {
 
 	return h.SearchEngine
 }
+
+func (h *MainHelper) SetReplicationLagForTesting(seconds int) error {
+	if dn := h.SQLStore.DriverName(); dn != model.DATABASE_DRIVER_MYSQL {
+		return fmt.Errorf("method not implemented for %q database driver, only %q is supported", dn, model.DATABASE_DRIVER_MYSQL)
+	}
+
+	err := h.execOnEachReplica("STOP SLAVE SQL_THREAD FOR CHANNEL ''")
+	if err != nil {
+		return err
+	}
+
+	err = h.execOnEachReplica(fmt.Sprintf("CHANGE MASTER TO MASTER_DELAY = %d", seconds))
+	if err != nil {
+		return err
+	}
+
+	err = h.execOnEachReplica("START SLAVE SQL_THREAD FOR CHANNEL ''")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h *MainHelper) execOnEachReplica(query string, args ...interface{}) error {
+	for _, replica := range h.SQLStore.Replicas {
+		_, err := replica.Exec(query, args...)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}