From 8429add371ecb67e232d8e8f0d09c9851ca31154 Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Mon, 17 Dec 2018 08:51:46 -0800 Subject: [PATCH] Cleanup related to context refactor (#9988) --- api4/apitestlib.go | 5 +- api4/file_test.go | 4 +- api4/oauth.go | 3 +- api4/plugin_test.go | 4 +- api4/system.go | 18 +- api4/system_test.go | 2 +- app/app.go | 53 +-- app/auto_posts.go | 3 +- app/config_test.go | 3 +- app/file_test.go | 4 +- app/helper_test.go | 8 +- app/import_functions_test.go | 12 +- app/import_test.go | 4 +- app/import_validators_test.go | 4 +- app/license.go | 10 + app/notification_email_test.go | 9 +- app/options.go | 8 +- app/plugin.go | 4 +- app/saml.go | 5 +- app/security_update_check.go | 28 +- app/server.go | 438 +++++++++--------- app/server_app_adapters.go | 169 +++++++ app/server_license.go | 11 + app/server_test.go | 21 +- app/timezone.go | 28 -- app/user.go | 3 +- cmd/mattermost/commands/config.go | 3 +- cmd/mattermost/commands/config_flag_test.go | 5 +- cmd/mattermost/commands/plugin_test.go | 3 +- cmd/mattermost/commands/server.go | 126 +---- cmd/mattermost/commands/server_test.go | 4 +- cmd/mattermost/commands/test.go | 4 +- cmd/platform/main.go | 6 +- jobs/jobs_watcher.go | 12 +- migrations/helper_test.go | 5 +- model/client4.go | 6 +- model/user.go | 3 +- .../timezones/default.go | 31 +- services/timezones/timezones.go | 55 +++ services/timezones/timezones_test.go | 18 + store/sqlstore/upgrade.go | 3 +- utils/config.go | 95 +--- utils/config_test.go | 289 ------------ utils/fileutils/fileutils.go | 99 ++++ utils/fileutils/fileutils_test.go | 293 ++++++++++++ utils/html.go | 3 +- utils/i18n.go | 3 +- utils/license.go | 3 +- utils/license_test.go | 4 +- utils/subpath.go | 3 +- utils/testutils/testutils.go | 4 +- utils/timezone.go | 25 - utils/utils.go | 3 +- web/static.go | 5 +- web/web_test.go | 2 +- 55 files changed, 1027 insertions(+), 949 deletions(-) create mode 100644 app/server_app_adapters.go create mode 100644 app/server_license.go delete mode 100644 app/timezone.go rename model/timezone.go => services/timezones/default.go (94%) create mode 100644 services/timezones/timezones.go create mode 100644 services/timezones/timezones_test.go create mode 100644 utils/fileutils/fileutils.go create mode 100644 utils/fileutils/fileutils_test.go delete mode 100644 utils/timezone.go diff --git a/api4/apitestlib.go b/api4/apitestlib.go index 709fdbc71c..04b6a42557 100644 --- a/api4/apitestlib.go +++ b/api4/apitestlib.go @@ -22,6 +22,7 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/mattermost/mattermost-server/web" "github.com/mattermost/mattermost-server/wsapi" @@ -63,7 +64,7 @@ func UseTestStore(store store.Store) { func setupTestHelper(enterprise bool, updateConfig func(*model.Config)) *TestHelper { testStore.DropAllTables() - permConfig, err := os.Open(utils.FindConfigFile("config.json")) + permConfig, err := os.Open(fileutils.FindConfigFile("config.json")) if err != nil { panic(err) } @@ -102,7 +103,7 @@ func setupTestHelper(enterprise bool, updateConfig func(*model.Config)) *TestHel if updateConfig != nil { th.App.UpdateConfig(updateConfig) } - serverErr := th.App.StartServer() + serverErr := th.Server.Start() if serverErr != nil { panic(serverErr) } diff --git a/api4/file_test.go b/api4/file_test.go index f82ba92935..4d2b7915b8 100644 --- a/api4/file_test.go +++ b/api4/file_test.go @@ -21,14 +21,14 @@ import ( "github.com/mattermost/mattermost-server/app" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/mattermost/mattermost-server/utils/testutils" ) var testDir = "" func init() { - testDir, _ = utils.FindDir("tests") + testDir, _ = fileutils.FindDir("tests") } func checkCond(tb testing.TB, cond bool, text string) { diff --git a/api4/oauth.go b/api4/oauth.go index 62791b3a4d..9f223d3208 100644 --- a/api4/oauth.go +++ b/api4/oauth.go @@ -13,6 +13,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func (api *API) InitOAuth() { @@ -392,7 +393,7 @@ func authorizeOAuthPage(c *Context, w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "text/html; charset=utf-8") w.Header().Set("Cache-Control", "no-cache, max-age=31556926, public") - staticDir, _ := utils.FindDir(model.CLIENT_DIR) + staticDir, _ := fileutils.FindDir(model.CLIENT_DIR) http.ServeFile(w, r, filepath.Join(staticDir, "root.html")) } diff --git a/api4/plugin_test.go b/api4/plugin_test.go index ed32a32a1f..4a143ee190 100644 --- a/api4/plugin_test.go +++ b/api4/plugin_test.go @@ -12,7 +12,7 @@ import ( "testing" "github.com/mattermost/mattermost-server/model" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/stretchr/testify/assert" ) @@ -38,7 +38,7 @@ func TestPlugin(t *testing.T) { *cfg.PluginSettings.EnableUploads = true }) - path, _ := utils.FindDir("tests") + path, _ := fileutils.FindDir("tests") tarData, err := ioutil.ReadFile(filepath.Join(path, "testplugin.tar.gz")) if err != nil { t.Fatal(err) diff --git a/api4/system.go b/api4/system.go index 26e69095a8..9fc748f071 100644 --- a/api4/system.go +++ b/api4/system.go @@ -5,6 +5,7 @@ package api4 import ( "bytes" + "encoding/json" "fmt" "io" "net/http" @@ -419,15 +420,18 @@ func getAnalytics(c *Context, w http.ResponseWriter, r *http.Request) { } func getSupportedTimezones(c *Context, w http.ResponseWriter, r *http.Request) { - supportedTimezones := c.App.Timezones() - - if supportedTimezones != nil { - w.Write([]byte(model.TimezonesToJson(supportedTimezones))) - return + supportedTimezones := c.App.Timezones.GetSupported() + if supportedTimezones == nil { + supportedTimezones = make([]string, 0) } - emptyTimezones := make([]string, 0) - w.Write([]byte(model.TimezonesToJson(emptyTimezones))) + b, err := json.Marshal(supportedTimezones) + if err != nil { + c.Log.Warn("Unable to marshal JSON in timezones.", mlog.Err(err)) + w.WriteHeader(http.StatusInternalServerError) + } + + w.Write(b) } func testS3(c *Context, w http.ResponseWriter, r *http.Request) { diff --git a/api4/system_test.go b/api4/system_test.go index 5828d36940..214c25a082 100644 --- a/api4/system_test.go +++ b/api4/system_test.go @@ -723,7 +723,7 @@ func TestSupportedTimezones(t *testing.T) { defer th.TearDown() Client := th.Client - supportedTimezonesFromConfig := th.App.Timezones() + supportedTimezonesFromConfig := th.App.Timezones.GetSupported() supportedTimezones, resp := Client.GetSupportedTimezone() CheckNoError(t, resp) diff --git a/app/app.go b/app/app.go index 2c527ed3a6..5dae4366e5 100644 --- a/app/app.go +++ b/app/app.go @@ -14,6 +14,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/services/httpservice" + "github.com/mattermost/mattermost-server/services/timezones" "github.com/mattermost/mattermost-server/utils" goi18n "github.com/nicksnyder/go-i18n/i18n" ) @@ -42,6 +43,7 @@ type App struct { Saml einterfaces.SamlInterface HTTPService httpservice.HTTPService + Timezones *timezones.Timezones } func New(options ...AppOption) *App { @@ -132,57 +134,6 @@ func (a *App) Handle404(w http.ResponseWriter, r *http.Request) { utils.RenderWebAppError(a.Config(), w, r, err, a.AsymmetricSigningKey()) } -func (a *App) StartElasticsearch() { - a.Srv.Go(func() { - if err := a.Elasticsearch.Start(); err != nil { - mlog.Error(err.Error()) - } - }) - - a.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) { - if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing { - a.Srv.Go(func() { - if err := a.Elasticsearch.Start(); err != nil { - mlog.Error(err.Error()) - } - }) - } else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing { - a.Srv.Go(func() { - if err := a.Elasticsearch.Stop(); err != nil { - mlog.Error(err.Error()) - } - }) - } else if *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff { - a.Srv.Go(func() { - if *oldConfig.ElasticsearchSettings.EnableIndexing { - if err := a.Elasticsearch.Stop(); err != nil { - mlog.Error(err.Error()) - } - if err := a.Elasticsearch.Start(); err != nil { - mlog.Error(err.Error()) - } - } - }) - } - }) - - a.AddLicenseListener(func() { - if a.License() != nil { - a.Srv.Go(func() { - if err := a.Elasticsearch.Start(); err != nil { - mlog.Error(err.Error()) - } - }) - } else { - a.Srv.Go(func() { - if err := a.Elasticsearch.Stop(); err != nil { - mlog.Error(err.Error()) - } - }) - } - }) -} - func (a *App) getSystemInstallDate() (int64, *model.AppError) { result := <-a.Srv.Store.System().GetByName(model.SYSTEM_INSTALLATION_DATE_KEY) if result.Err != nil { diff --git a/app/auto_posts.go b/app/auto_posts.go index 23746a9ba9..0eda016678 100644 --- a/app/auto_posts.go +++ b/app/auto_posts.go @@ -11,6 +11,7 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) type AutoPostCreator struct { @@ -43,7 +44,7 @@ func NewAutoPostCreator(client *model.Client4, channelid string) *AutoPostCreato func (cfg *AutoPostCreator) UploadTestFile() ([]string, bool) { filename := cfg.ImageFilenames[utils.RandIntFromRange(utils.Range{Begin: 0, End: len(cfg.ImageFilenames) - 1})] - path, _ := utils.FindDir("web/static/images") + path, _ := fileutils.FindDir("web/static/images") file, err := os.Open(filepath.Join(path, filename)) if err != nil { return nil, false diff --git a/app/config_test.go b/app/config_test.go index a885eb62f3..271305c60e 100644 --- a/app/config_test.go +++ b/app/config_test.go @@ -16,13 +16,14 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store/sqlstore" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func TestLoadConfig(t *testing.T) { tempConfig, err := ioutil.TempFile("", "") require.Nil(t, err) - input, err := ioutil.ReadFile(utils.FindConfigFile("config.json")) + input, err := ioutil.ReadFile(fileutils.FindConfigFile("config.json")) require.Nil(t, err) lines := strings.Split(string(input), "\n") for i, line := range lines { diff --git a/app/file_test.go b/app/file_test.go index 577ca809a9..ab803d3ec7 100644 --- a/app/file_test.go +++ b/app/file_test.go @@ -14,7 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/mattermost/mattermost-server/model" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func TestGeneratePublicLinkHash(t *testing.T) { @@ -171,7 +171,7 @@ func TestMigrateFilenamesToFileInfos(t *testing.T) { infos = th.App.MigrateFilenamesToFileInfos(post) assert.Equal(t, 0, len(infos)) - path, _ := utils.FindDir("tests") + path, _ := fileutils.FindDir("tests") file, fileErr := os.Open(filepath.Join(path, "test.png")) require.Nil(t, fileErr) defer file.Close() diff --git a/app/helper_test.go b/app/helper_test.go index d8f5218837..8b0facc896 100644 --- a/app/helper_test.go +++ b/app/helper_test.go @@ -15,6 +15,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) type TestHelper struct { @@ -35,7 +36,7 @@ type TestHelper struct { func setupTestHelper(enterprise bool) *TestHelper { mainHelper.Store.DropAllTables() - permConfig, err := os.Open(utils.FindConfigFile("config.json")) + permConfig, err := os.Open(fileutils.FindConfigFile("config.json")) if err != nil { panic(err) } @@ -68,16 +69,13 @@ func setupTestHelper(enterprise bool) *TestHelper { th.App.UpdateConfig(func(cfg *model.Config) { *cfg.RateLimitSettings.Enable = false }) prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" }) - serverErr := th.App.StartServer() + serverErr := th.Server.Start() if serverErr != nil { panic(serverErr) } th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = prevListenAddress }) - th.App.DoAdvancedPermissionsMigration() - th.App.DoEmojisPermissionsMigration() - th.App.Srv.Store.MarkSystemRanUnitTests() th.App.UpdateConfig(func(cfg *model.Config) { *cfg.TeamSettings.EnableOpenServer = true }) diff --git a/app/import_functions_test.go b/app/import_functions_test.go index 31708c6c32..586e43ae91 100644 --- a/app/import_functions_test.go +++ b/app/import_functions_test.go @@ -13,7 +13,7 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/store" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func TestImportImportScheme(t *testing.T) { @@ -613,7 +613,7 @@ func TestImportImportUser(t *testing.T) { // Do a valid user in apply mode. username := model.NewId() - testsDir, _ := utils.FindDir("tests") + testsDir, _ := fileutils.FindDir("tests") data = UserImportData{ ProfileImage: ptrStr(filepath.Join(testsDir, "test.png")), Username: &username, @@ -2345,7 +2345,7 @@ func TestImportImportEmoji(t *testing.T) { th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.EnableCustomEmoji = true }) - testsDir, _ := utils.FindDir("tests") + testsDir, _ := fileutils.FindDir("tests") testImage := filepath.Join(testsDir, "test.png") data := EmojiImportData{Name: ptrStr(model.NewId())} @@ -2382,7 +2382,7 @@ func TestImportAttachment(t *testing.T) { th := Setup() defer th.TearDown() - testsDir, _ := utils.FindDir("tests") + testsDir, _ := fileutils.FindDir("tests") testImage := filepath.Join(testsDir, "test.png") invalidPath := "some-invalid-path" @@ -2455,7 +2455,7 @@ func TestImportPostAndRepliesWithAttachments(t *testing.T) { time := model.GetMillis() attachmentsPostTime := time attachmentsReplyTime := time + 1 - testsDir, _ := utils.FindDir("tests") + testsDir, _ := fileutils.FindDir("tests") testImage := filepath.Join(testsDir, "test.png") testMarkDown := filepath.Join(testsDir, "test-attachments.md") data := &PostImportData{ @@ -2545,7 +2545,7 @@ func TestImportDirectPostWithAttachments(t *testing.T) { th := Setup() defer th.TearDown() - testsDir, _ := utils.FindDir("tests") + testsDir, _ := fileutils.FindDir("tests") testImage := filepath.Join(testsDir, "test.png") // Create a user. diff --git a/app/import_test.go b/app/import_test.go index 89c8344e67..9087177a3b 100644 --- a/app/import_test.go +++ b/app/import_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/mattermost/mattermost-server/model" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func ptrStr(s string) *string { @@ -171,7 +171,7 @@ func TestImportBulkImport(t *testing.T) { username2 := model.NewId() username3 := model.NewId() emojiName := model.NewId() - testsDir, _ := utils.FindDir("tests") + testsDir, _ := fileutils.FindDir("tests") testImage := filepath.Join(testsDir, "test.png") teamTheme1 := `{\"awayIndicator\":\"#DBBD4E\",\"buttonBg\":\"#23A1FF\",\"buttonColor\":\"#FFFFFF\",\"centerChannelBg\":\"#ffffff\",\"centerChannelColor\":\"#333333\",\"codeTheme\":\"github\",\"image\":\"/static/files/a4a388b38b32678e83823ef1b3e17766.png\",\"linkColor\":\"#2389d7\",\"mentionBg\":\"#2389d7\",\"mentionColor\":\"#ffffff\",\"mentionHighlightBg\":\"#fff2bb\",\"mentionHighlightLink\":\"#2f81b7\",\"newMessageSeparator\":\"#FF8800\",\"onlineIndicator\":\"#7DBE00\",\"sidebarBg\":\"#fafafa\",\"sidebarHeaderBg\":\"#3481B9\",\"sidebarHeaderTextColor\":\"#ffffff\",\"sidebarText\":\"#333333\",\"sidebarTextActiveBorder\":\"#378FD2\",\"sidebarTextActiveColor\":\"#111111\",\"sidebarTextHoverBg\":\"#e6f2fa\",\"sidebarUnreadText\":\"#333333\",\"type\":\"Mattermost\"}` teamTheme2 := `{\"awayIndicator\":\"#DBBD4E\",\"buttonBg\":\"#23A100\",\"buttonColor\":\"#EEEEEE\",\"centerChannelBg\":\"#ffffff\",\"centerChannelColor\":\"#333333\",\"codeTheme\":\"github\",\"image\":\"/static/files/a4a388b38b32678e83823ef1b3e17766.png\",\"linkColor\":\"#2389d7\",\"mentionBg\":\"#2389d7\",\"mentionColor\":\"#ffffff\",\"mentionHighlightBg\":\"#fff2bb\",\"mentionHighlightLink\":\"#2f81b7\",\"newMessageSeparator\":\"#FF8800\",\"onlineIndicator\":\"#7DBE00\",\"sidebarBg\":\"#fafafa\",\"sidebarHeaderBg\":\"#3481B9\",\"sidebarHeaderTextColor\":\"#ffffff\",\"sidebarText\":\"#333333\",\"sidebarTextActiveBorder\":\"#378FD2\",\"sidebarTextActiveColor\":\"#222222\",\"sidebarTextHoverBg\":\"#e6f2fa\",\"sidebarUnreadText\":\"#444444\",\"type\":\"Mattermost\"}` diff --git a/app/import_validators_test.go b/app/import_validators_test.go index 8ec4607783..08ef674e0c 100644 --- a/app/import_validators_test.go +++ b/app/import_validators_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/mattermost/mattermost-server/model" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/stretchr/testify/assert" ) @@ -572,7 +572,7 @@ func TestImportValidateUserImportData(t *testing.T) { } // Test a valid User with all fields populated. - testsDir, _ := utils.FindDir("tests") + testsDir, _ := fileutils.FindDir("tests") data = UserImportData{ ProfileImage: ptrStr(filepath.Join(testsDir, "test.png")), Username: ptrStr("bob"), diff --git a/app/license.go b/app/license.go index 72779a98ad..1b8e0ad283 100644 --- a/app/license.go +++ b/app/license.go @@ -171,12 +171,22 @@ func (a *App) RemoveLicense() *model.AppError { return nil } +func (s *Server) AddLicenseListener(listener func()) string { + id := model.NewId() + s.licenseListeners[id] = listener + return id +} + func (a *App) AddLicenseListener(listener func()) string { id := model.NewId() a.Srv.licenseListeners[id] = listener return id } +func (s *Server) RemoveLicenseListener(id string) { + delete(s.licenseListeners, id) +} + func (a *App) RemoveLicenseListener(id string) { delete(a.Srv.licenseListeners, id) } diff --git a/app/notification_email_test.go b/app/notification_email_test.go index d45ca424ad..c32cd42489 100644 --- a/app/notification_email_test.go +++ b/app/notification_email_test.go @@ -11,6 +11,7 @@ import ( "time" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/timezones" "github.com/mattermost/mattermost-server/utils" ) @@ -230,7 +231,7 @@ func TestGetNotificationEmailBodyFullNotificationLocaleTimeWithTimezone(t *testi defer th.TearDown() recipient := &model.User{ - Timezone: model.DefaultUserTimezone(), + Timezone: timezones.DefaultUserTimezone(), } recipient.Timezone["automaticTimezone"] = "America/New_York" post := &model.Post{ @@ -261,7 +262,7 @@ func TestGetNotificationEmailBodyFullNotificationLocaleTimeNoTimezone(t *testing defer th.TearDown() recipient := &model.User{ - Timezone: model.DefaultUserTimezone(), + Timezone: timezones.DefaultUserTimezone(), } post := &model.Post{ CreateAt: 1524681000000, @@ -303,7 +304,7 @@ func TestGetNotificationEmailBodyFullNotificationLocaleTime12Hour(t *testing.T) defer th.TearDown() recipient := &model.User{ - Timezone: model.DefaultUserTimezone(), + Timezone: timezones.DefaultUserTimezone(), } recipient.Timezone["automaticTimezone"] = "America/New_York" post := &model.Post{ @@ -335,7 +336,7 @@ func TestGetNotificationEmailBodyFullNotificationLocaleTime24Hour(t *testing.T) defer th.TearDown() recipient := &model.User{ - Timezone: model.DefaultUserTimezone(), + Timezone: timezones.DefaultUserTimezone(), } recipient.Timezone["automaticTimezone"] = "America/New_York" post := &model.Post{ diff --git a/app/options.go b/app/options.go index 3abf133c1f..77fe99d640 100644 --- a/app/options.go +++ b/app/options.go @@ -36,6 +36,10 @@ func ConfigFile(file string) Option { } } +func RunJobs(s *Server) { + s.runjobs = true +} + func DisableConfigWatch(s *Server) { s.disableConfigWatch = true } @@ -49,7 +53,6 @@ func ServerConnector(s *Server) AppOption { a.Log = s.Log - a.HTTPService = s.HTTPService a.AccountMigration = s.AccountMigration a.Cluster = s.Cluster a.Compliance = s.Compliance @@ -59,5 +62,8 @@ func ServerConnector(s *Server) AppOption { a.MessageExport = s.MessageExport a.Metrics = s.Metrics a.Saml = s.Saml + + a.HTTPService = s.HTTPService + a.Timezones = s.timezones } } diff --git a/app/plugin.go b/app/plugin.go index 962d153412..d58afd6c30 100644 --- a/app/plugin.go +++ b/app/plugin.go @@ -12,7 +12,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/plugin" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) // GetPluginsEnvironment returns the plugin environment for use if plugins are enabled and @@ -146,7 +146,7 @@ func (a *App) InitPlugins(pluginDir, webappPluginDir string) { } a.SetPluginsEnvironment(env) - prepackagedPluginsDir, found := utils.FindDir("prepackaged_plugins") + prepackagedPluginsDir, found := fileutils.FindDir("prepackaged_plugins") if found { if err := filepath.Walk(prepackagedPluginsDir, func(walkPath string, info os.FileInfo, err error) error { if !strings.HasSuffix(walkPath, ".tar.gz") { diff --git a/app/saml.go b/app/saml.go index c81e4f674e..f721ccd6ec 100644 --- a/app/saml.go +++ b/app/saml.go @@ -12,6 +12,7 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func (a *App) GetSamlMetadata() (string, *model.AppError) { @@ -40,7 +41,7 @@ func WriteSamlFile(fileData *multipart.FileHeader) *model.AppError { } defer file.Close() - configDir, _ := utils.FindDir("config") + configDir, _ := fileutils.FindDir("config") out, err := os.Create(filepath.Join(configDir, filename)) if err != nil { return model.NewAppError("AddSamlCertificate", "api.admin.add_certificate.saving.app_error", nil, err.Error(), http.StatusInternalServerError) @@ -112,7 +113,7 @@ func RemoveSamlFile(filename string) *model.AppError { return model.NewAppError("AddSamlCertificate", "api.admin.remove_certificate.delete.app_error", nil, "", http.StatusBadRequest) } - if err := os.Remove(utils.FindConfigFile(filename)); err != nil { + if err := os.Remove(fileutils.FindConfigFile(filename)); err != nil { return model.NewAppError("removeCertificate", "api.admin.remove_certificate.delete.app_error", map[string]interface{}{"Filename": filename}, err.Error(), http.StatusInternalServerError) } diff --git a/app/security_update_check.go b/app/security_update_check.go index 39c1a6ee95..d9eacdc102 100644 --- a/app/security_update_check.go +++ b/app/security_update_check.go @@ -13,6 +13,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/mailservice" "github.com/mattermost/mattermost-server/utils" ) @@ -31,9 +32,9 @@ const ( PROP_SECURITY_UNIT_TESTS = "ut" ) -func (a *App) DoSecurityUpdateCheck() { - if *a.Config().ServiceSettings.EnableSecurityFixAlert { - if result := <-a.Srv.Store.System().Get(); result.Err == nil { +func (s *Server) DoSecurityUpdateCheck() { + if *s.Config().ServiceSettings.EnableSecurityFixAlert { + if result := <-s.Store.System().Get(); result.Err == nil { props := result.Data.(model.StringMap) lastSecurityTime, _ := strconv.ParseInt(props[model.SYSTEM_LAST_SECURITY_TIME], 10, 0) currentTime := model.GetMillis() @@ -43,10 +44,10 @@ func (a *App) DoSecurityUpdateCheck() { v := url.Values{} - v.Set(PROP_SECURITY_ID, a.DiagnosticId()) + v.Set(PROP_SECURITY_ID, s.diagnosticId) v.Set(PROP_SECURITY_BUILD, model.CurrentVersion+"."+model.BuildNumber) v.Set(PROP_SECURITY_ENTERPRISE_READY, model.BuildEnterpriseReady) - v.Set(PROP_SECURITY_DATABASE, *a.Config().SqlSettings.DriverName) + v.Set(PROP_SECURITY_DATABASE, *s.Config().SqlSettings.DriverName) v.Set(PROP_SECURITY_OS, runtime.GOOS) if len(props[model.SYSTEM_RAN_UNIT_TESTS]) > 0 { @@ -57,20 +58,20 @@ func (a *App) DoSecurityUpdateCheck() { systemSecurityLastTime := &model.System{Name: model.SYSTEM_LAST_SECURITY_TIME, Value: strconv.FormatInt(currentTime, 10)} if lastSecurityTime == 0 { - <-a.Srv.Store.System().Save(systemSecurityLastTime) + <-s.Store.System().Save(systemSecurityLastTime) } else { - <-a.Srv.Store.System().Update(systemSecurityLastTime) + <-s.Store.System().Update(systemSecurityLastTime) } - if ucr := <-a.Srv.Store.User().GetTotalUsersCount(); ucr.Err == nil { + if ucr := <-s.Store.User().GetTotalUsersCount(); ucr.Err == nil { v.Set(PROP_SECURITY_USER_COUNT, strconv.FormatInt(ucr.Data.(int64), 10)) } - if ucr := <-a.Srv.Store.Status().GetTotalActiveUsersCount(); ucr.Err == nil { + if ucr := <-s.Store.Status().GetTotalActiveUsersCount(); ucr.Err == nil { v.Set(PROP_SECURITY_ACTIVE_USER_COUNT, strconv.FormatInt(ucr.Data.(int64), 10)) } - if tcr := <-a.Srv.Store.Team().AnalyticsTeamCount(); tcr.Err == nil { + if tcr := <-s.Store.Team().AnalyticsTeamCount(); tcr.Err == nil { v.Set(PROP_SECURITY_TEAM_COUNT, strconv.FormatInt(tcr.Data.(int64), 10)) } @@ -86,7 +87,7 @@ func (a *App) DoSecurityUpdateCheck() { for _, bulletin := range bulletins { if bulletin.AppliesToVersion == model.CurrentVersion { if props["SecurityBulletin_"+bulletin.Id] == "" { - results := <-a.Srv.Store.User().GetSystemAdminProfiles() + results := <-s.Store.User().GetSystemAdminProfiles() if results.Err != nil { mlog.Error("Failed to get system admins for security update information from Mattermost.") return @@ -108,11 +109,12 @@ func (a *App) DoSecurityUpdateCheck() { for _, user := range users { mlog.Info(fmt.Sprintf("Sending security bulletin for %v to %v", bulletin.Id, user.Email)) - a.SendMail(user.Email, utils.T("mattermost.bulletin.subject"), string(body)) + license := s.License() + mailservice.SendMailUsingConfig(user.Email, utils.T("mattermost.bulletin.subject"), string(body), s.Config(), license != nil && *license.Features.Compliance) } bulletinSeen := &model.System{Name: "SecurityBulletin_" + bulletin.Id, Value: bulletin.Id} - <-a.Srv.Store.System().Save(bulletinSeen) + <-s.Store.System().Save(bulletinSeen) } } } diff --git a/app/server.go b/app/server.go index ba817cc1d1..8684ef4bc5 100644 --- a/app/server.go +++ b/app/server.go @@ -14,7 +14,6 @@ import ( "net/http" "net/url" "os" - "path" "strings" "sync" "sync/atomic" @@ -33,10 +32,10 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/plugin" "github.com/mattermost/mattermost-server/services/httpservice" - "github.com/mattermost/mattermost-server/services/mailservice" + "github.com/mattermost/mattermost-server/services/timezones" "github.com/mattermost/mattermost-server/store" - "github.com/mattermost/mattermost-server/store/sqlstore" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) var MaxNotificationsPerChannelDefault int64 = 1000000 @@ -73,7 +72,8 @@ type Server struct { PushNotificationsHub PushNotificationsHub - Jobs *jobs.JobServer + runjobs bool + Jobs *jobs.JobServer config atomic.Value envConfig map[string]interface{} @@ -85,7 +85,7 @@ type Server struct { clientLicenseValue atomic.Value licenseListeners map[string]func() - timezones atomic.Value + timezones *timezones.Timezones newStore func() store.Store @@ -124,150 +124,6 @@ type Server struct { Saml einterfaces.SamlInterface } -// This is a bridge between the old and new initalization for the context refactor. -// It calls app layer initalization code that then turns around and acts on the server. -// Don't add anything new here, new initilization should be done in the server and -// performed in the NewServer function. -func (s *Server) RunOldAppInitalization() error { - a := s.FakeApp() - - a.CreatePushNotificationsHub() - a.StartPushNotificationsHubWorkers() - - if utils.T == nil { - if err := utils.TranslationsPreInit(); err != nil { - return errors.Wrapf(err, "unable to load Mattermost translation files") - } - } - model.AppErrorInit(utils.T) - - a.LoadTimezones() - - if err := utils.InitTranslations(a.Config().LocalizationSettings); err != nil { - return errors.Wrapf(err, "unable to load Mattermost translation files") - } - - a.Srv.configListenerId = a.AddConfigListener(func(_, _ *model.Config) { - a.configOrLicenseListener() - - message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_CONFIG_CHANGED, "", "", "", nil) - - message.Add("config", a.ClientConfigWithComputed()) - a.Srv.Go(func() { - a.Publish(message) - }) - }) - a.Srv.licenseListenerId = a.AddLicenseListener(func() { - a.configOrLicenseListener() - - message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_LICENSE_CHANGED, "", "", "", nil) - message.Add("license", a.GetSanitizedClientLicense()) - a.Srv.Go(func() { - a.Publish(message) - }) - - }) - - if err := a.SetupInviteEmailRateLimiting(); err != nil { - return err - } - - mlog.Info("Server is initializing...") - - s.initEnterprise() - - if a.Srv.newStore == nil { - a.Srv.newStore = func() store.Store { - return store.NewLayeredStore(sqlstore.NewSqlSupplier(a.Config().SqlSettings, a.Metrics), a.Metrics, a.Cluster) - } - } - - if htmlTemplateWatcher, err := utils.NewHTMLTemplateWatcher("templates"); err != nil { - mlog.Error(fmt.Sprintf("Failed to parse server templates %v", err)) - } else { - a.Srv.htmlTemplateWatcher = htmlTemplateWatcher - } - - a.Srv.Store = a.Srv.newStore() - - if err := a.ensureAsymmetricSigningKey(); err != nil { - return errors.Wrapf(err, "unable to ensure asymmetric signing key") - } - - if err := a.ensureInstallationDate(); err != nil { - return errors.Wrapf(err, "unable to ensure installation date") - } - - a.EnsureDiagnosticId() - a.regenerateClientConfig() - - s.initJobs() - a.AddLicenseListener(func() { - s.initJobs() - }) - - a.Srv.clusterLeaderListenerId = a.AddClusterLeaderChangedListener(func() { - mlog.Info("Cluster leader changed. Determining if job schedulers should be running:", mlog.Bool("isLeader", a.IsLeader())) - a.Srv.Jobs.Schedulers.HandleClusterLeaderChange(a.IsLeader()) - }) - - subpath, err := utils.GetSubpathFromConfig(a.Config()) - if err != nil { - return errors.Wrap(err, "failed to parse SiteURL subpath") - } - a.Srv.Router = a.Srv.RootRouter.PathPrefix(subpath).Subrouter() - a.Srv.Router.HandleFunc("/plugins/{plugin_id:[A-Za-z0-9\\_\\-\\.]+}", a.ServePluginRequest) - a.Srv.Router.HandleFunc("/plugins/{plugin_id:[A-Za-z0-9\\_\\-\\.]+}/{anything:.*}", a.ServePluginRequest) - - // If configured with a subpath, redirect 404s at the root back into the subpath. - if subpath != "/" { - a.Srv.RootRouter.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r.URL.Path = path.Join(subpath, r.URL.Path) - http.Redirect(w, r, r.URL.String(), http.StatusFound) - }) - } - a.Srv.Router.NotFoundHandler = http.HandlerFunc(a.Handle404) - - a.Srv.WebSocketRouter = &WebSocketRouter{ - app: a, - handlers: make(map[string]webSocketHandler), - } - - mailservice.TestConnection(a.Config()) - - if _, err := url.ParseRequestURI(*a.Config().ServiceSettings.SiteURL); err != nil { - mlog.Error("SiteURL must be set. Some features will operate incorrectly if the SiteURL is not set. See documentation for details: http://about.mattermost.com/default-site-url") - } - - backend, appErr := a.FileBackend() - if appErr == nil { - appErr = backend.TestConnection() - } - if appErr != nil { - mlog.Error("Problem with file storage settings: " + appErr.Error()) - } - - if model.BuildEnterpriseReady == "true" { - a.LoadLicense() - } - - a.DoAdvancedPermissionsMigration() - a.DoEmojisPermissionsMigration() - - a.InitPostMetadata() - - a.InitPlugins(*a.Config().PluginSettings.Directory, *a.Config().PluginSettings.ClientDirectory) - a.AddConfigListener(func(prevCfg, cfg *model.Config) { - if *cfg.PluginSettings.Enable { - a.InitPlugins(*cfg.PluginSettings.Directory, *a.Config().PluginSettings.ClientDirectory) - } else { - a.ShutDownPlugins() - } - }) - - return nil -} - func NewServer(options ...Option) (*Server, error) { rootRouter := mux.NewRouter() @@ -305,11 +161,21 @@ func NewServer(options ...Option) (*Server, error) { s.HTTPService = httpservice.MakeHTTPService(s.FakeApp()) + if utils.T == nil { + if err := utils.TranslationsPreInit(); err != nil { + return nil, errors.Wrapf(err, "unable to load Mattermost translation files") + } + } + err := s.RunOldAppInitalization() if err != nil { return nil, err } + model.AppErrorInit(utils.T) + + s.timezones = timezones.New("") + // Start email batching because it's not like the other jobs s.InitEmailBatching() s.AddConfigListener(func(_, _ *model.Config) { @@ -320,7 +186,7 @@ func NewServer(options ...Option) (*Server, error) { mlog.Info(fmt.Sprintf("Enterprise Enabled: %v", model.BuildEnterpriseReady)) pwd, _ := os.Getwd() mlog.Info(fmt.Sprintf("Current working directory is %v", pwd)) - mlog.Info(fmt.Sprintf("Loaded config file from %v", utils.FindConfigFile(s.configFile))) + mlog.Info(fmt.Sprintf("Loaded config file from %v", fileutils.FindConfigFile(s.configFile))) license := s.License() @@ -348,6 +214,50 @@ func NewServer(options ...Option) (*Server, error) { mlog.Error(fmt.Sprint("Error to reset the server status.", result.Err.Error())) } + if s.Cluster != nil { + s.FakeApp().RegisterAllClusterMessageHandlers() + s.Cluster.StartInterNodeCommunication() + } + + if s.Metrics != nil { + s.Metrics.StartServer() + } + + if s.Elasticsearch != nil { + s.StartElasticsearch() + } + + s.initJobs() + + if s.runjobs { + s.Go(func() { + runSecurityJob(s) + }) + s.Go(func() { + runDiagnosticsJob(s) + }) + s.Go(func() { + runSessionCleanupJob(s) + }) + s.Go(func() { + runTokenCleanupJob(s) + }) + s.Go(func() { + runCommandWebhookCleanupJob(s) + }) + + if complianceI := s.Compliance; complianceI != nil { + complianceI.StartComplianceDailyJob() + } + + if *s.Config().JobSettings.RunJobs && s.Jobs != nil { + s.Jobs.StartWorkers() + } + if *s.Config().JobSettings.RunScheduler && s.Jobs != nil { + s.Jobs.StartSchedulers() + } + } + return s, nil } @@ -358,19 +268,6 @@ func (s *Server) AppOptions() []AppOption { } } -// A temporary bridge to deal with cases where the code is so tighly coupled that -// this is easier as a temporary solution -func (s *Server) FakeApp() *App { - a := New( - ServerConnector(s), - ) - return a -} - -func (s *Server) StartServer() error { - return s.FakeApp().StartServer() -} - const TIME_TO_WAIT_FOR_CONNECTIONS_TO_CLOSE_ON_SERVER_SHUTDOWN = time.Second func (s *Server) StopHTTPServer() { @@ -395,15 +292,6 @@ func (s *Server) StopHTTPServer() { } } -func (s *Server) RunOldAppShutdown() { - a := s.FakeApp() - a.HubStop() - a.StopPushNotificationsHubWorkers() - a.ShutDownPlugins() - a.RemoveLicenseListener(s.licenseListenerId) - a.RemoveClusterLeaderChangedListener(s.clusterLeaderListenerId) -} - func (s *Server) Shutdown() error { mlog.Info("Stopping Server...") @@ -425,15 +313,23 @@ func (s *Server) Shutdown() error { s.DisableConfigWatch() + if s.Cluster != nil { + s.Cluster.StopInterNodeCommunication() + } + + if s.Metrics != nil { + s.Metrics.StopServer() + } + + if s.Jobs != nil && s.runjobs { + s.Jobs.StopWorkers() + s.Jobs.StopSchedulers() + } + mlog.Info("Server stopped") return nil } -func (s *Server) License() *model.License { - license, _ := s.licenseValue.Load().(*model.License) - return license -} - // Go creates a goroutine, but maintains a record of it to ensure that execution completes before // the server is shutdown. func (s *Server) Go(f func()) { @@ -493,14 +389,14 @@ func stripPort(hostport string) string { return net.JoinHostPort(host, "443") } -func (a *App) StartServer() error { +func (s *Server) Start() error { mlog.Info("Starting Server...") - var handler http.Handler = a.Srv.RootRouter - if allowedOrigins := *a.Config().ServiceSettings.AllowCorsFrom; allowedOrigins != "" { - exposedCorsHeaders := *a.Config().ServiceSettings.CorsExposedHeaders - allowCredentials := *a.Config().ServiceSettings.CorsAllowCredentials - debug := *a.Config().ServiceSettings.CorsDebug + var handler http.Handler = s.RootRouter + if allowedOrigins := *s.Config().ServiceSettings.AllowCorsFrom; allowedOrigins != "" { + exposedCorsHeaders := *s.Config().ServiceSettings.CorsExposedHeaders + allowCredentials := *s.Config().ServiceSettings.CorsAllowCredentials + debug := *s.Config().ServiceSettings.CorsDebug corsWrapper := cors.New(cors.Options{ AllowedOrigins: strings.Fields(allowedOrigins), AllowedMethods: corsAllowedMethods, @@ -513,34 +409,34 @@ func (a *App) StartServer() error { // If we have debugging of CORS turned on then forward messages to logs if debug { - corsWrapper.Log = a.Log.StdLog(mlog.String("source", "cors")) + corsWrapper.Log = s.Log.StdLog(mlog.String("source", "cors")) } handler = corsWrapper.Handler(handler) } - if *a.Config().RateLimitSettings.Enable { + if *s.Config().RateLimitSettings.Enable { mlog.Info("RateLimiter is enabled") - rateLimiter, err := NewRateLimiter(&a.Config().RateLimitSettings) + rateLimiter, err := NewRateLimiter(&s.Config().RateLimitSettings) if err != nil { return err } - a.Srv.RateLimiter = rateLimiter + s.RateLimiter = rateLimiter handler = rateLimiter.RateLimitHandler(handler) } - a.Srv.Server = &http.Server{ + s.Server = &http.Server{ Handler: handlers.RecoveryHandler(handlers.RecoveryLogger(&RecoveryLogger{}), handlers.PrintRecoveryStack(true))(handler), - ReadTimeout: time.Duration(*a.Config().ServiceSettings.ReadTimeout) * time.Second, - WriteTimeout: time.Duration(*a.Config().ServiceSettings.WriteTimeout) * time.Second, - ErrorLog: a.Log.StdLog(mlog.String("source", "httpserver")), + ReadTimeout: time.Duration(*s.Config().ServiceSettings.ReadTimeout) * time.Second, + WriteTimeout: time.Duration(*s.Config().ServiceSettings.WriteTimeout) * time.Second, + ErrorLog: s.Log.StdLog(mlog.String("source", "httpserver")), } - addr := *a.Config().ServiceSettings.ListenAddress + addr := *s.Config().ServiceSettings.ListenAddress if addr == "" { - if *a.Config().ServiceSettings.ConnectionSecurity == model.CONN_SECURITY_TLS { + if *s.Config().ServiceSettings.ConnectionSecurity == model.CONN_SECURITY_TLS { addr = ":https" } else { addr = ":http" @@ -552,23 +448,23 @@ func (a *App) StartServer() error { errors.Wrapf(err, utils.T("api.server.start_server.starting.critical"), err) return err } - a.Srv.ListenAddr = listener.Addr().(*net.TCPAddr) + s.ListenAddr = listener.Addr().(*net.TCPAddr) mlog.Info(fmt.Sprintf("Server is listening on %v", listener.Addr().String())) // Migration from old let's encrypt library - if *a.Config().ServiceSettings.UseLetsEncrypt { - if stat, err := os.Stat(*a.Config().ServiceSettings.LetsEncryptCertificateCacheFile); err == nil && !stat.IsDir() { - os.Remove(*a.Config().ServiceSettings.LetsEncryptCertificateCacheFile) + if *s.Config().ServiceSettings.UseLetsEncrypt { + if stat, err := os.Stat(*s.Config().ServiceSettings.LetsEncryptCertificateCacheFile); err == nil && !stat.IsDir() { + os.Remove(*s.Config().ServiceSettings.LetsEncryptCertificateCacheFile) } } m := &autocert.Manager{ - Cache: autocert.DirCache(*a.Config().ServiceSettings.LetsEncryptCertificateCacheFile), + Cache: autocert.DirCache(*s.Config().ServiceSettings.LetsEncryptCertificateCacheFile), Prompt: autocert.AcceptTOS, } - if *a.Config().ServiceSettings.Forward80To443 { + if *s.Config().ServiceSettings.Forward80To443 { if host, port, err := net.SplitHostPort(addr); err != nil { mlog.Error("Unable to setup forwarding: " + err.Error()) } else if port != "443" { @@ -576,11 +472,11 @@ func (a *App) StartServer() error { } else { httpListenAddress := net.JoinHostPort(host, "http") - if *a.Config().ServiceSettings.UseLetsEncrypt { + if *s.Config().ServiceSettings.UseLetsEncrypt { server := &http.Server{ Addr: httpListenAddress, Handler: m.HTTPHandler(nil), - ErrorLog: a.Log.StdLog(mlog.String("source", "le_forwarder_server")), + ErrorLog: s.Log.StdLog(mlog.String("source", "le_forwarder_server")), } go server.ListenAndServe() } else { @@ -594,27 +490,27 @@ func (a *App) StartServer() error { server := &http.Server{ Handler: http.HandlerFunc(handleHTTPRedirect), - ErrorLog: a.Log.StdLog(mlog.String("source", "forwarder_server")), + ErrorLog: s.Log.StdLog(mlog.String("source", "forwarder_server")), } server.Serve(redirectListener) }() } } - } else if *a.Config().ServiceSettings.UseLetsEncrypt { + } else if *s.Config().ServiceSettings.UseLetsEncrypt { return errors.New(utils.T("api.server.start_server.forward80to443.disabled_while_using_lets_encrypt")) } - a.Srv.didFinishListen = make(chan struct{}) + s.didFinishListen = make(chan struct{}) go func() { var err error - if *a.Config().ServiceSettings.ConnectionSecurity == model.CONN_SECURITY_TLS { + if *s.Config().ServiceSettings.ConnectionSecurity == model.CONN_SECURITY_TLS { tlsConfig := &tls.Config{ PreferServerCipherSuites: true, CurvePreferences: []tls.CurveID{tls.CurveP521, tls.CurveP384, tls.CurveP256}, } - switch *a.Config().ServiceSettings.TLSMinVer { + switch *s.Config().ServiceSettings.TLSMinVer { case "1.0": tlsConfig.MinVersion = tls.VersionTLS10 case "1.1": @@ -632,11 +528,11 @@ func (a *App) StartServer() error { tls.TLS_RSA_WITH_AES_256_GCM_SHA384, } - if len(a.Config().ServiceSettings.TLSOverwriteCiphers) == 0 { + if len(s.Config().ServiceSettings.TLSOverwriteCiphers) == 0 { tlsConfig.CipherSuites = defaultCiphers } else { var cipherSuites []uint16 - for _, cipher := range a.Config().ServiceSettings.TLSOverwriteCiphers { + for _, cipher := range s.Config().ServiceSettings.TLSOverwriteCiphers { value, ok := model.ServerTLSSupportedCiphers[cipher] if !ok { @@ -658,18 +554,18 @@ func (a *App) StartServer() error { certFile := "" keyFile := "" - if *a.Config().ServiceSettings.UseLetsEncrypt { + if *s.Config().ServiceSettings.UseLetsEncrypt { tlsConfig.GetCertificate = m.GetCertificate tlsConfig.NextProtos = append(tlsConfig.NextProtos, "h2") } else { - certFile = *a.Config().ServiceSettings.TLSCertFile - keyFile = *a.Config().ServiceSettings.TLSKeyFile + certFile = *s.Config().ServiceSettings.TLSCertFile + keyFile = *s.Config().ServiceSettings.TLSKeyFile } - a.Srv.Server.TLSConfig = tlsConfig - err = a.Srv.Server.ServeTLS(listener, certFile, keyFile) + s.Server.TLSConfig = tlsConfig + err = s.Server.ServeTLS(listener, certFile, keyFile) } else { - err = a.Srv.Server.Serve(listener) + err = s.Server.Serve(listener) } if err != nil && err != http.ErrServerClosed { @@ -677,7 +573,7 @@ func (a *App) StartServer() error { time.Sleep(time.Second) } - close(a.Srv.didFinishListen) + close(s.didFinishListen) }() return nil @@ -705,3 +601,115 @@ func consumeAndClose(r *http.Response) { r.Body.Close() } } + +func runSecurityJob(s *Server) { + doSecurity(s) + model.CreateRecurringTask("Security", func() { + doSecurity(s) + }, time.Hour*4) +} + +func runDiagnosticsJob(s *Server) { + doDiagnostics(s) + model.CreateRecurringTask("Diagnostics", func() { + doDiagnostics(s) + }, time.Hour*24) +} + +func runTokenCleanupJob(s *Server) { + doTokenCleanup(s) + model.CreateRecurringTask("Token Cleanup", func() { + doTokenCleanup(s) + }, time.Hour*1) +} + +func runCommandWebhookCleanupJob(s *Server) { + doCommandWebhookCleanup(s) + model.CreateRecurringTask("Command Hook Cleanup", func() { + doCommandWebhookCleanup(s) + }, time.Hour*1) +} + +func runSessionCleanupJob(s *Server) { + doSessionCleanup(s) + model.CreateRecurringTask("Session Cleanup", func() { + doSessionCleanup(s) + }, time.Hour*24) +} + +func doSecurity(s *Server) { + s.DoSecurityUpdateCheck() +} + +func doDiagnostics(s *Server) { + if *s.Config().LogSettings.EnableDiagnostics { + s.FakeApp().SendDailyDiagnostics() + } +} + +func doTokenCleanup(s *Server) { + s.Store.Token().Cleanup() +} + +func doCommandWebhookCleanup(s *Server) { + s.Store.CommandWebhook().Cleanup() +} + +const ( + SESSIONS_CLEANUP_BATCH_SIZE = 1000 +) + +func doSessionCleanup(s *Server) { + s.Store.Session().Cleanup(model.GetMillis(), SESSIONS_CLEANUP_BATCH_SIZE) +} + +func (s *Server) StartElasticsearch() { + s.Go(func() { + if err := s.Elasticsearch.Start(); err != nil { + s.Log.Error(err.Error()) + } + }) + + s.AddConfigListener(func(oldConfig *model.Config, newConfig *model.Config) { + if !*oldConfig.ElasticsearchSettings.EnableIndexing && *newConfig.ElasticsearchSettings.EnableIndexing { + s.Go(func() { + if err := s.Elasticsearch.Start(); err != nil { + mlog.Error(err.Error()) + } + }) + } else if *oldConfig.ElasticsearchSettings.EnableIndexing && !*newConfig.ElasticsearchSettings.EnableIndexing { + s.Go(func() { + if err := s.Elasticsearch.Stop(); err != nil { + mlog.Error(err.Error()) + } + }) + } else if *oldConfig.ElasticsearchSettings.Password != *newConfig.ElasticsearchSettings.Password || *oldConfig.ElasticsearchSettings.Username != *newConfig.ElasticsearchSettings.Username || *oldConfig.ElasticsearchSettings.ConnectionUrl != *newConfig.ElasticsearchSettings.ConnectionUrl || *oldConfig.ElasticsearchSettings.Sniff != *newConfig.ElasticsearchSettings.Sniff { + s.Go(func() { + if *oldConfig.ElasticsearchSettings.EnableIndexing { + if err := s.Elasticsearch.Stop(); err != nil { + mlog.Error(err.Error()) + } + if err := s.Elasticsearch.Start(); err != nil { + mlog.Error(err.Error()) + } + } + }) + } + }) + + s.AddLicenseListener(func() { + if s.License() != nil { + s.Go(func() { + if err := s.Elasticsearch.Start(); err != nil { + mlog.Error(err.Error()) + } + }) + } else { + s.Go(func() { + if err := s.Elasticsearch.Stop(); err != nil { + mlog.Error(err.Error()) + } + }) + } + }) +} diff --git a/app/server_app_adapters.go b/app/server_app_adapters.go new file mode 100644 index 0000000000..86e93317e1 --- /dev/null +++ b/app/server_app_adapters.go @@ -0,0 +1,169 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import ( + "fmt" + "net/http" + "net/url" + "path" + + "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/mailservice" + "github.com/mattermost/mattermost-server/store" + "github.com/mattermost/mattermost-server/store/sqlstore" + "github.com/mattermost/mattermost-server/utils" + "github.com/pkg/errors" +) + +// This is a bridge between the old and new initalization for the context refactor. +// It calls app layer initalization code that then turns around and acts on the server. +// Don't add anything new here, new initilization should be done in the server and +// performed in the NewServer function. +func (s *Server) RunOldAppInitalization() error { + a := s.FakeApp() + + a.CreatePushNotificationsHub() + a.StartPushNotificationsHubWorkers() + + if err := utils.InitTranslations(a.Config().LocalizationSettings); err != nil { + return errors.Wrapf(err, "unable to load Mattermost translation files") + } + + a.Srv.configListenerId = a.AddConfigListener(func(_, _ *model.Config) { + a.configOrLicenseListener() + + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_CONFIG_CHANGED, "", "", "", nil) + + message.Add("config", a.ClientConfigWithComputed()) + a.Srv.Go(func() { + a.Publish(message) + }) + }) + a.Srv.licenseListenerId = a.AddLicenseListener(func() { + a.configOrLicenseListener() + + message := model.NewWebSocketEvent(model.WEBSOCKET_EVENT_LICENSE_CHANGED, "", "", "", nil) + message.Add("license", a.GetSanitizedClientLicense()) + a.Srv.Go(func() { + a.Publish(message) + }) + + }) + + if err := a.SetupInviteEmailRateLimiting(); err != nil { + return err + } + + mlog.Info("Server is initializing...") + + s.initEnterprise() + + if a.Srv.newStore == nil { + a.Srv.newStore = func() store.Store { + return store.NewLayeredStore(sqlstore.NewSqlSupplier(a.Config().SqlSettings, a.Metrics), a.Metrics, a.Cluster) + } + } + + if htmlTemplateWatcher, err := utils.NewHTMLTemplateWatcher("templates"); err != nil { + mlog.Error(fmt.Sprintf("Failed to parse server templates %v", err)) + } else { + a.Srv.htmlTemplateWatcher = htmlTemplateWatcher + } + + a.Srv.Store = a.Srv.newStore() + + if err := a.ensureAsymmetricSigningKey(); err != nil { + return errors.Wrapf(err, "unable to ensure asymmetric signing key") + } + + if err := a.ensureInstallationDate(); err != nil { + return errors.Wrapf(err, "unable to ensure installation date") + } + + a.EnsureDiagnosticId() + a.regenerateClientConfig() + + a.Srv.clusterLeaderListenerId = a.AddClusterLeaderChangedListener(func() { + mlog.Info("Cluster leader changed. Determining if job schedulers should be running:", mlog.Bool("isLeader", a.IsLeader())) + if a.Srv.Jobs != nil { + a.Srv.Jobs.Schedulers.HandleClusterLeaderChange(a.IsLeader()) + } + }) + + subpath, err := utils.GetSubpathFromConfig(a.Config()) + if err != nil { + return errors.Wrap(err, "failed to parse SiteURL subpath") + } + a.Srv.Router = a.Srv.RootRouter.PathPrefix(subpath).Subrouter() + a.Srv.Router.HandleFunc("/plugins/{plugin_id:[A-Za-z0-9\\_\\-\\.]+}", a.ServePluginRequest) + a.Srv.Router.HandleFunc("/plugins/{plugin_id:[A-Za-z0-9\\_\\-\\.]+}/{anything:.*}", a.ServePluginRequest) + + // If configured with a subpath, redirect 404s at the root back into the subpath. + if subpath != "/" { + a.Srv.RootRouter.NotFoundHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.URL.Path = path.Join(subpath, r.URL.Path) + http.Redirect(w, r, r.URL.String(), http.StatusFound) + }) + } + a.Srv.Router.NotFoundHandler = http.HandlerFunc(a.Handle404) + + a.Srv.WebSocketRouter = &WebSocketRouter{ + app: a, + handlers: make(map[string]webSocketHandler), + } + + mailservice.TestConnection(a.Config()) + + if _, err := url.ParseRequestURI(*a.Config().ServiceSettings.SiteURL); err != nil { + mlog.Error("SiteURL must be set. Some features will operate incorrectly if the SiteURL is not set. See documentation for details: http://about.mattermost.com/default-site-url") + } + + backend, appErr := a.FileBackend() + if appErr == nil { + appErr = backend.TestConnection() + } + if appErr != nil { + mlog.Error("Problem with file storage settings: " + appErr.Error()) + } + + if model.BuildEnterpriseReady == "true" { + a.LoadLicense() + } + + a.DoAdvancedPermissionsMigration() + a.DoEmojisPermissionsMigration() + + a.InitPostMetadata() + + a.InitPlugins(*a.Config().PluginSettings.Directory, *a.Config().PluginSettings.ClientDirectory) + a.AddConfigListener(func(prevCfg, cfg *model.Config) { + if *cfg.PluginSettings.Enable { + a.InitPlugins(*cfg.PluginSettings.Directory, *a.Config().PluginSettings.ClientDirectory) + } else { + a.ShutDownPlugins() + } + }) + + return nil +} + +func (s *Server) RunOldAppShutdown() { + a := s.FakeApp() + a.HubStop() + a.StopPushNotificationsHubWorkers() + a.ShutDownPlugins() + a.RemoveLicenseListener(s.licenseListenerId) + a.RemoveClusterLeaderChangedListener(s.clusterLeaderListenerId) +} + +// A temporary bridge to deal with cases where the code is so tighly coupled that +// this is easier as a temporary solution +func (s *Server) FakeApp() *App { + a := New( + ServerConnector(s), + ) + return a +} diff --git a/app/server_license.go b/app/server_license.go new file mode 100644 index 0000000000..5b14ab865f --- /dev/null +++ b/app/server_license.go @@ -0,0 +1,11 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package app + +import "github.com/mattermost/mattermost-server/model" + +func (s *Server) License() *model.License { + license, _ := s.licenseValue.Load().(*model.License) + return license +} diff --git a/app/server_test.go b/app/server_test.go index 581c20c826..8755b9ecb7 100644 --- a/app/server_test.go +++ b/app/server_test.go @@ -12,9 +12,8 @@ import ( "strings" "testing" - "github.com/mattermost/mattermost-server/utils" - "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/stretchr/testify/require" ) @@ -23,7 +22,7 @@ func TestStartServerSuccess(t *testing.T) { require.NoError(t, err) s.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" }) - serverErr := s.StartServer() + serverErr := s.Start() client := &http.Client{} checkEndpoint(t, client, "http://localhost:"+strconv.Itoa(s.ListenAddr.Port)+"/", http.StatusNotFound) @@ -42,7 +41,7 @@ func TestStartServerRateLimiterCriticalError(t *testing.T) { *cfg.RateLimitSettings.MaxBurst = -100 }) - serverErr := s.StartServer() + serverErr := s.Start() s.Shutdown() require.Error(t, serverErr) } @@ -60,7 +59,7 @@ func TestStartServerPortUnavailable(t *testing.T) { *cfg.ServiceSettings.ListenAddress = listener.Addr().String() }) - serverErr := s.StartServer() + serverErr := s.Start() s.Shutdown() require.Error(t, serverErr) } @@ -69,14 +68,14 @@ func TestStartServerTLSSuccess(t *testing.T) { s, err := NewServer() require.NoError(t, err) - testDir, _ := utils.FindDir("tests") + testDir, _ := fileutils.FindDir("tests") s.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" *cfg.ServiceSettings.ConnectionSecurity = "TLS" *cfg.ServiceSettings.TLSKeyFile = path.Join(testDir, "tls_test_key.pem") *cfg.ServiceSettings.TLSCertFile = path.Join(testDir, "tls_test_cert.pem") }) - serverErr := s.StartServer() + serverErr := s.Start() tr := &http.Transport{ TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, @@ -93,7 +92,7 @@ func TestStartServerTLSVersion(t *testing.T) { s, err := NewServer() require.NoError(t, err) - testDir, _ := utils.FindDir("tests") + testDir, _ := fileutils.FindDir("tests") s.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" *cfg.ServiceSettings.ConnectionSecurity = "TLS" @@ -101,7 +100,7 @@ func TestStartServerTLSVersion(t *testing.T) { *cfg.ServiceSettings.TLSKeyFile = path.Join(testDir, "tls_test_key.pem") *cfg.ServiceSettings.TLSCertFile = path.Join(testDir, "tls_test_cert.pem") }) - serverErr := s.StartServer() + serverErr := s.Start() tr := &http.Transport{ TLSClientConfig: &tls.Config{ @@ -137,7 +136,7 @@ func TestStartServerTLSOverwriteCipher(t *testing.T) { s, err := NewServer() require.NoError(t, err) - testDir, _ := utils.FindDir("tests") + testDir, _ := fileutils.FindDir("tests") s.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" *cfg.ServiceSettings.ConnectionSecurity = "TLS" @@ -148,7 +147,7 @@ func TestStartServerTLSOverwriteCipher(t *testing.T) { *cfg.ServiceSettings.TLSKeyFile = path.Join(testDir, "tls_test_key.pem") *cfg.ServiceSettings.TLSCertFile = path.Join(testDir, "tls_test_cert.pem") }) - serverErr := s.StartServer() + serverErr := s.Start() tr := &http.Transport{ TLSClientConfig: &tls.Config{ diff --git a/app/timezone.go b/app/timezone.go deleted file mode 100644 index e014eb025c..0000000000 --- a/app/timezone.go +++ /dev/null @@ -1,28 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package app - -import ( - "github.com/mattermost/mattermost-server/model" - "github.com/mattermost/mattermost-server/utils" -) - -func (a *App) Timezones() model.SupportedTimezones { - if cfg := a.Srv.timezones.Load(); cfg != nil { - return cfg.(model.SupportedTimezones) - } - return model.SupportedTimezones{} -} - -func (a *App) LoadTimezones() { - timezonePath := "timezones.json" - - if a.Config().TimezoneSettings.SupportedTimezonesPath != nil && len(*a.Config().TimezoneSettings.SupportedTimezonesPath) > 0 { - timezonePath = *a.Config().TimezoneSettings.SupportedTimezonesPath - } - - timezoneCfg := utils.LoadTimezones(timezonePath) - - a.Srv.timezones.Store(timezoneCfg) -} diff --git a/app/user.go b/app/user.go index 4f8a89e457..2fd399bea2 100644 --- a/app/user.go +++ b/app/user.go @@ -30,6 +30,7 @@ import ( "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/services/mfa" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) const ( @@ -707,7 +708,7 @@ func getFont(initialFont string) (*truetype.Font, error) { initialFont = "nunito-bold.ttf" } - fontDir, _ := utils.FindDir("fonts") + fontDir, _ := fileutils.FindDir("fonts") fontBytes, err := ioutil.ReadFile(filepath.Join(fontDir, initialFont)) if err != nil { return nil, err diff --git a/cmd/mattermost/commands/config.go b/cmd/mattermost/commands/config.go index f74c25f323..7d1afa85d8 100644 --- a/cmd/mattermost/commands/config.go +++ b/cmd/mattermost/commands/config.go @@ -18,6 +18,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) var ConfigCmd = &cobra.Command{ @@ -89,7 +90,7 @@ func configValidateCmdF(command *cobra.Command, args []string) error { return err } - filePath = utils.FindConfigFile(filePath) + filePath = fileutils.FindConfigFile(filePath) file, err := os.Open(filePath) if err != nil { diff --git a/cmd/mattermost/commands/config_flag_test.go b/cmd/mattermost/commands/config_flag_test.go index 1ccbb9685e..b5004a772b 100644 --- a/cmd/mattermost/commands/config_flag_test.go +++ b/cmd/mattermost/commands/config_flag_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func TestConfigFlag(t *testing.T) { @@ -20,12 +21,12 @@ func TestConfigFlag(t *testing.T) { defer th.TearDown() dir := th.TemporaryDirectory() - timezones := utils.LoadTimezones("timezones.json") + timezones := th.App.Timezones.GetSupported() tzConfigPath := filepath.Join(dir, "timezones.json") timezoneData, _ := json.Marshal(timezones) require.NoError(t, ioutil.WriteFile(tzConfigPath, timezoneData, 0600)) - i18n, ok := utils.FindDir("i18n") + i18n, ok := fileutils.FindDir("i18n") require.True(t, ok) require.NoError(t, utils.CopyDir(i18n, filepath.Join(dir, "i18n"))) diff --git a/cmd/mattermost/commands/plugin_test.go b/cmd/mattermost/commands/plugin_test.go index e3c8fe1843..3c932559a1 100644 --- a/cmd/mattermost/commands/plugin_test.go +++ b/cmd/mattermost/commands/plugin_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -23,7 +24,7 @@ func TestPlugin(t *testing.T) { os.MkdirAll("./test-plugins", os.ModePerm) os.MkdirAll("./test-client-plugins", os.ModePerm) - path, _ := utils.FindDir("tests") + path, _ := fileutils.FindDir("tests") os.Chdir(filepath.Join("..", "..", "..")) diff --git a/cmd/mattermost/commands/server.go b/cmd/mattermost/commands/server.go index d6a88db061..90246756d1 100644 --- a/cmd/mattermost/commands/server.go +++ b/cmd/mattermost/commands/server.go @@ -8,22 +8,16 @@ import ( "os" "os/signal" "syscall" - "time" "github.com/mattermost/mattermost-server/api4" "github.com/mattermost/mattermost-server/app" "github.com/mattermost/mattermost-server/manualtesting" "github.com/mattermost/mattermost-server/mlog" - "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/web" "github.com/mattermost/mattermost-server/wsapi" "github.com/spf13/cobra" ) -const ( - SESSIONS_CLEANUP_BATCH_SIZE = 1000 -) - var serverCmd = &cobra.Command{ Use: "server", Short: "Run the Mattermost server", @@ -50,7 +44,10 @@ func serverCmdF(command *cobra.Command, args []string) error { } func runServer(configFileLocation string, disableConfigWatch bool, usedPlatform bool, interruptChan chan os.Signal) error { - options := []app.Option{app.ConfigFile(configFileLocation)} + options := []app.Option{ + app.ConfigFile(configFileLocation), + app.RunJobs, + } if disableConfigWatch { options = append(options, app.DisableConfigWatch) } @@ -61,69 +58,25 @@ func runServer(configFileLocation string, disableConfigWatch bool, usedPlatform } defer server.Shutdown() - a := server.FakeApp() - if usedPlatform { mlog.Error("The platform binary has been deprecated, please switch to using the mattermost binary.") } - serverErr := a.StartServer() + serverErr := server.Start() if serverErr != nil { mlog.Critical(serverErr.Error()) return serverErr } api := api4.Init(server, server.AppOptions, server.Router) - wsapi.Init(a, server.WebSocketRouter) + wsapi.Init(server.FakeApp(), server.WebSocketRouter) web.New(server, server.AppOptions, server.Router) // If we allow testing then listen for manual testing URL hits - if a.Config().ServiceSettings.EnableTesting { + if server.Config().ServiceSettings.EnableTesting { manualtesting.Init(api) } - a.Srv.Go(func() { - runSecurityJob(a) - }) - a.Srv.Go(func() { - runDiagnosticsJob(a) - }) - a.Srv.Go(func() { - runSessionCleanupJob(a) - }) - a.Srv.Go(func() { - runTokenCleanupJob(a) - }) - a.Srv.Go(func() { - runCommandWebhookCleanupJob(a) - }) - - if complianceI := a.Compliance; complianceI != nil { - complianceI.StartComplianceDailyJob() - } - - if a.Cluster != nil { - a.RegisterAllClusterMessageHandlers() - a.Cluster.StartInterNodeCommunication() - } - - if a.Metrics != nil { - a.Metrics.StartServer() - } - - if a.Elasticsearch != nil { - a.StartElasticsearch() - } - - if *a.Config().JobSettings.RunJobs { - a.Srv.Jobs.StartWorkers() - defer a.Srv.Jobs.StopWorkers() - } - if *a.Config().JobSettings.RunScheduler { - a.Srv.Jobs.StartSchedulers() - defer a.Srv.Jobs.StopSchedulers() - } - notifyReady() // wait for kill signal before attempting to gracefully shutdown @@ -131,62 +84,9 @@ func runServer(configFileLocation string, disableConfigWatch bool, usedPlatform signal.Notify(interruptChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) <-interruptChan - if a.Cluster != nil { - a.Cluster.StopInterNodeCommunication() - } - - if a.Metrics != nil { - a.Metrics.StopServer() - } - return nil } -func runSecurityJob(a *app.App) { - doSecurity(a) - model.CreateRecurringTask("Security", func() { - doSecurity(a) - }, time.Hour*4) -} - -func runDiagnosticsJob(a *app.App) { - doDiagnostics(a) - model.CreateRecurringTask("Diagnostics", func() { - doDiagnostics(a) - }, time.Hour*24) -} - -func runTokenCleanupJob(a *app.App) { - doTokenCleanup(a) - model.CreateRecurringTask("Token Cleanup", func() { - doTokenCleanup(a) - }, time.Hour*1) -} - -func runCommandWebhookCleanupJob(a *app.App) { - doCommandWebhookCleanup(a) - model.CreateRecurringTask("Command Hook Cleanup", func() { - doCommandWebhookCleanup(a) - }, time.Hour*1) -} - -func runSessionCleanupJob(a *app.App) { - doSessionCleanup(a) - model.CreateRecurringTask("Session Cleanup", func() { - doSessionCleanup(a) - }, time.Hour*24) -} - -func doSecurity(a *app.App) { - a.DoSecurityUpdateCheck() -} - -func doDiagnostics(a *app.App) { - if *a.Config().LogSettings.EnableDiagnostics { - a.SendDailyDiagnostics() - } -} - func notifyReady() { // If the environment vars provide a systemd notification socket, // notify systemd that the server is ready. @@ -215,15 +115,3 @@ func sendSystemdReadyNotification(socketPath string) error { _, err = conn.Write([]byte(msg)) return err } - -func doTokenCleanup(a *app.App) { - a.Srv.Store.Token().Cleanup() -} - -func doCommandWebhookCleanup(a *app.App) { - a.Srv.Store.CommandWebhook().Cleanup() -} - -func doSessionCleanup(a *app.App) { - a.Srv.Store.Session().Cleanup(model.GetMillis(), SESSIONS_CLEANUP_BATCH_SIZE) -} diff --git a/cmd/mattermost/commands/server_test.go b/cmd/mattermost/commands/server_test.go index 0f825e316c..57dbd45cfc 100644 --- a/cmd/mattermost/commands/server_test.go +++ b/cmd/mattermost/commands/server_test.go @@ -11,7 +11,7 @@ import ( "testing" "github.com/mattermost/mattermost-server/jobs" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/stretchr/testify/require" ) @@ -36,7 +36,7 @@ func SetupServerTest() *ServerTestHelper { jobs.DEFAULT_WATCHER_POLLING_INTERVAL = 200 th := &ServerTestHelper{ - configPath: utils.FindConfigFile("config.json"), + configPath: fileutils.FindConfigFile("config.json"), disableConfigWatch: true, interruptChan: interruptChan, originalInterval: originalInterval, diff --git a/cmd/mattermost/commands/test.go b/cmd/mattermost/commands/test.go index e1a6fcd784..22abb6d5a6 100644 --- a/cmd/mattermost/commands/test.go +++ b/cmd/mattermost/commands/test.go @@ -53,7 +53,7 @@ func webClientTestsCmdF(command *cobra.Command, args []string) error { defer a.Shutdown() utils.InitTranslations(a.Config().LocalizationSettings) - serverErr := a.StartServer() + serverErr := a.Srv.Start() if serverErr != nil { return serverErr } @@ -74,7 +74,7 @@ func serverForWebClientTestsCmdF(command *cobra.Command, args []string) error { defer a.Shutdown() utils.InitTranslations(a.Config().LocalizationSettings) - serverErr := a.StartServer() + serverErr := a.Srv.Start() if serverErr != nil { return serverErr } diff --git a/cmd/platform/main.go b/cmd/platform/main.go index 25e091a84f..13605a34f6 100644 --- a/cmd/platform/main.go +++ b/cmd/platform/main.go @@ -8,7 +8,7 @@ import ( "os" "syscall" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func main() { @@ -25,9 +25,9 @@ The platform binary will be removed in a future version. args[0] = "mattermost" args = append(args, "--platform") - realMattermost := utils.FindFile("mattermost") + realMattermost := fileutils.FindFile("mattermost") if realMattermost == "" { - realMattermost = utils.FindFile("bin/mattermost") + realMattermost = fileutils.FindFile("bin/mattermost") } if realMattermost == "" { diff --git a/jobs/jobs_watcher.go b/jobs/jobs_watcher.go index 01d0a8d0f7..1fecc17550 100644 --- a/jobs/jobs_watcher.go +++ b/jobs/jobs_watcher.go @@ -20,15 +20,15 @@ type Watcher struct { srv *JobServer workers *Workers - stop chan bool - stopped chan bool + stop chan struct{} + stopped chan struct{} pollingInterval int } func (srv *JobServer) MakeWatcher(workers *Workers, pollingInterval int) *Watcher { return &Watcher{ - stop: make(chan bool, 1), - stopped: make(chan bool, 1), + stop: make(chan struct{}), + stopped: make(chan struct{}), pollingInterval: pollingInterval, workers: workers, srv: srv, @@ -45,7 +45,7 @@ func (watcher *Watcher) Start() { defer func() { mlog.Debug("Watcher Finished") - watcher.stopped <- true + close(watcher.stopped) }() for { @@ -61,7 +61,7 @@ func (watcher *Watcher) Start() { func (watcher *Watcher) Stop() { mlog.Debug("Watcher Stopping") - watcher.stop <- true + close(watcher.stop) <-watcher.stopped } diff --git a/migrations/helper_test.go b/migrations/helper_test.go index 586de27cf2..653a40c716 100644 --- a/migrations/helper_test.go +++ b/migrations/helper_test.go @@ -13,6 +13,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) type TestHelper struct { @@ -33,7 +34,7 @@ type TestHelper struct { func setupTestHelper(enterprise bool) *TestHelper { mainHelper.Store.DropAllTables() - permConfig, err := os.Open(utils.FindConfigFile("config.json")) + permConfig, err := os.Open(fileutils.FindConfigFile("config.json")) if err != nil { panic(err) } @@ -67,7 +68,7 @@ func setupTestHelper(enterprise bool) *TestHelper { prevListenAddress := *th.App.Config().ServiceSettings.ListenAddress th.App.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" }) - serverErr := th.App.StartServer() + serverErr := th.Server.Start() if serverErr != nil { panic(serverErr) } diff --git a/model/client4.go b/model/client4.go index f57a51f7f7..da8c14ccd1 100644 --- a/model/client4.go +++ b/model/client4.go @@ -3595,13 +3595,15 @@ func (c *Client4) DeleteReaction(reaction *Reaction) (bool, *Response) { // Timezone Section // GetSupportedTimezone returns a page of supported timezones on the system. -func (c *Client4) GetSupportedTimezone() (SupportedTimezones, *Response) { +func (c *Client4) GetSupportedTimezone() ([]string, *Response) { r, err := c.DoApiGet(c.GetTimezonesRoute(), "") if err != nil { return nil, BuildErrorResponse(r, err) } defer closeBody(r) - return TimezonesFromJson(r.Body), BuildResponse(r) + var timezones []string + json.NewDecoder(r.Body).Decode(&timezones) + return timezones, BuildResponse(r) } // Open Graph Metadata Section diff --git a/model/user.go b/model/user.go index 43caf93a69..6b9bfc93dc 100644 --- a/model/user.go +++ b/model/user.go @@ -12,6 +12,7 @@ import ( "strings" "unicode/utf8" + "github.com/mattermost/mattermost-server/services/timezones" "golang.org/x/crypto/bcrypt" ) @@ -230,7 +231,7 @@ func (u *User) PreSave() { } if u.Timezone == nil { - u.Timezone = DefaultUserTimezone() + u.Timezone = timezones.DefaultUserTimezone() } if len(u.Password) > 0 { diff --git a/model/timezone.go b/services/timezones/default.go similarity index 94% rename from model/timezone.go rename to services/timezones/default.go index 420b9d2e2f..065d0eef72 100644 --- a/model/timezone.go +++ b/services/timezones/default.go @@ -1,34 +1,7 @@ -// Copyright (c) 2017-present Mattermost, Inc. All Rights Reserved. +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. // See License.txt for license information. -package model - -import ( - "encoding/json" - "io" -) - -type SupportedTimezones []string - -func TimezonesToJson(timezoneList []string) string { - b, _ := json.Marshal(timezoneList) - return string(b) -} - -func TimezonesFromJson(data io.Reader) SupportedTimezones { - var timezones SupportedTimezones - json.NewDecoder(data).Decode(&timezones) - return timezones -} - -func DefaultUserTimezone() map[string]string { - defaultTimezone := make(map[string]string) - defaultTimezone["useAutomaticTimezone"] = "true" - defaultTimezone["automaticTimezone"] = "" - defaultTimezone["manualTimezone"] = "" - - return defaultTimezone -} +package timezones var DefaultSupportedTimezones = []string{ "Africa/Abidjan", diff --git a/services/timezones/timezones.go b/services/timezones/timezones.go new file mode 100644 index 0000000000..0f5c7425e6 --- /dev/null +++ b/services/timezones/timezones.go @@ -0,0 +1,55 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package timezones + +import ( + "encoding/json" + "io/ioutil" + "sync/atomic" + + "github.com/mattermost/mattermost-server/utils/fileutils" +) + +type Timezones struct { + supportedZones atomic.Value +} + +func New(timezonesConfigFile string) *Timezones { + timezones := Timezones{} + + if len(timezonesConfigFile) == 0 { + timezonesConfigFile = "timezones.json" + } + + var supportedTimezones []string + + // Attempt to get timezones from config. Failure results in defaults. + if timezoneFile := fileutils.FindConfigFile(timezonesConfigFile); timezoneFile == "" { + supportedTimezones = DefaultSupportedTimezones + } else if raw, err := ioutil.ReadFile(timezoneFile); err != nil { + supportedTimezones = DefaultSupportedTimezones + } else if err := json.Unmarshal(raw, &supportedTimezones); err != nil { + supportedTimezones = DefaultSupportedTimezones + } + + timezones.supportedZones.Store(supportedTimezones) + + return &timezones +} + +func (t *Timezones) GetSupported() []string { + if supportedZonesValue := t.supportedZones.Load(); supportedZonesValue != nil { + return supportedZonesValue.([]string) + } + return []string{} +} + +func DefaultUserTimezone() map[string]string { + defaultTimezone := make(map[string]string) + defaultTimezone["useAutomaticTimezone"] = "true" + defaultTimezone["automaticTimezone"] = "" + defaultTimezone["manualTimezone"] = "" + + return defaultTimezone +} diff --git a/services/timezones/timezones_test.go b/services/timezones/timezones_test.go new file mode 100644 index 0000000000..83a56d9302 --- /dev/null +++ b/services/timezones/timezones_test.go @@ -0,0 +1,18 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package timezones + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestTimezoneConfig(t *testing.T) { + tz1 := New("timezones.json") + assert.NotEmpty(t, tz1.GetSupported()) + + tz2 := New("timezones_file_does_not_exists.json") + assert.Equal(t, DefaultSupportedTimezones, tz2.GetSupported()) +} diff --git a/store/sqlstore/upgrade.go b/store/sqlstore/upgrade.go index 35944f99ef..5ae458145e 100644 --- a/store/sqlstore/upgrade.go +++ b/store/sqlstore/upgrade.go @@ -12,6 +12,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/services/timezones" ) const ( @@ -416,7 +417,7 @@ func UpgradeDatabaseToVersion49(sqlStore SqlStore) { if shouldPerformUpgrade(sqlStore, VERSION_4_8_1, VERSION_4_9_0) { sqlStore.CreateColumnIfNotExists("Teams", "LastTeamIconUpdate", "bigint", "bigint", "0") - defaultTimezone := model.DefaultUserTimezone() + defaultTimezone := timezones.DefaultUserTimezone() defaultTimezoneValue, err := json.Marshal(defaultTimezone) if err != nil { mlog.Critical(fmt.Sprint(err)) diff --git a/utils/config.go b/utils/config.go index f5c49b9b8a..6b392b22db 100644 --- a/utils/config.go +++ b/utils/config.go @@ -24,6 +24,7 @@ import ( "github.com/mattermost/mattermost-server/einterfaces" "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/mattermost/mattermost-server/utils/jsonutils" ) @@ -33,13 +34,6 @@ const ( ) var ( - commonBaseSearchPaths = []string{ - ".", - "..", - "../..", - "../../..", - } - termsOfServiceEnabledAndEmpty = model.NewAppError( "Config.IsValid", "model.config.is_valid.support.custom_terms_of_service_text.app_error", @@ -49,87 +43,6 @@ var ( ) ) -func FindPath(path string, baseSearchPaths []string, filter func(os.FileInfo) bool) string { - if filepath.IsAbs(path) { - if _, err := os.Stat(path); err == nil { - return path - } - - return "" - } - - searchPaths := []string{} - searchPaths = append(searchPaths, baseSearchPaths...) - - // Additionally attempt to search relative to the location of the running binary. - var binaryDir string - if exe, err := os.Executable(); err == nil { - if exe, err = filepath.EvalSymlinks(exe); err == nil { - if exe, err = filepath.Abs(exe); err == nil { - binaryDir = filepath.Dir(exe) - } - } - } - if binaryDir != "" { - for _, baseSearchPath := range baseSearchPaths { - searchPaths = append( - searchPaths, - filepath.Join(binaryDir, baseSearchPath), - ) - } - } - - for _, parent := range searchPaths { - found, err := filepath.Abs(filepath.Join(parent, path)) - if err != nil { - continue - } else if fileInfo, err := os.Stat(found); err == nil { - if filter != nil { - if filter(fileInfo) { - return found - } - } else { - return found - } - } - } - - return "" -} - -// FindConfigFile attempts to find an existing configuration file. fileName can be an absolute or -// relative path or name such as "/opt/mattermost/config.json" or simply "config.json". An empty -// string is returned if no configuration is found. -func FindConfigFile(fileName string) (path string) { - found := FindFile(filepath.Join("config", fileName)) - if found == "" { - found = FindPath(fileName, []string{"."}, nil) - } - - return found -} - -// FindFile looks for the given file in nearby ancestors relative to the current working -// directory as well as the directory of the executable. -func FindFile(path string) string { - return FindPath(path, commonBaseSearchPaths, func(fileInfo os.FileInfo) bool { - return !fileInfo.IsDir() - }) -} - -// FindDir looks for the given directory in nearby ancestors relative to the current working -// directory as well as the directory of the executable, falling back to `./` if not found. -func FindDir(dir string) (string, bool) { - found := FindPath(dir, commonBaseSearchPaths, func(fileInfo os.FileInfo) bool { - return fileInfo.IsDir() - }) - if found == "" { - return "./", false - } - - return found, true -} - func MloggerConfigFromLoggerConfig(s *model.LogSettings) *mlog.LoggerConfiguration { return &mlog.LoggerConfiguration{ EnableConsole: s.EnableConsole, @@ -154,7 +67,7 @@ func EnableDebugLogForTest() { func GetLogFileLocation(fileLocation string) string { if fileLocation == "" { - fileLocation, _ = FindDir("logs") + fileLocation, _ = fileutils.FindDir("logs") } return filepath.Join(fileLocation, LOG_FILENAME) @@ -432,10 +345,10 @@ func ReadConfigFile(path string, allowEnvironmentOverrides bool) (*model.Config, // it will attempt to locate a default config file, and copy it to a file named fileName in the same // directory. In either case, the config file path is returned. func EnsureConfigFile(fileName string) (string, error) { - if configFile := FindConfigFile(fileName); configFile != "" { + if configFile := fileutils.FindConfigFile(fileName); configFile != "" { return configFile, nil } - if defaultPath := FindConfigFile("default.json"); defaultPath != "" { + if defaultPath := fileutils.FindConfigFile("default.json"); defaultPath != "" { destPath := filepath.Join(filepath.Dir(defaultPath), fileName) src, err := os.Open(defaultPath) if err != nil { diff --git a/utils/config_test.go b/utils/config_test.go index 77705192dd..74b886e36c 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -6,9 +6,7 @@ package utils import ( "bytes" "fmt" - "io/ioutil" "os" - "path/filepath" "strings" "testing" @@ -112,293 +110,6 @@ func TestReadConfig_PluginSettings(t *testing.T) { } } -func TestTimezoneConfig(t *testing.T) { - TranslationsPreInit() - supportedTimezones := LoadTimezones("timezones.json") - assert.Equal(t, len(supportedTimezones) > 0, true) - - supportedTimezones2 := LoadTimezones("timezones_file_does_not_exists.json") - assert.Equal(t, len(supportedTimezones2) > 0, true) -} - -func TestFindConfigFile(t *testing.T) { - t.Run("config.json in current working directory, not inside config/", func(t *testing.T) { - // Force a unique working directory - cwd, err := ioutil.TempDir("", "") - require.NoError(t, err) - defer os.RemoveAll(cwd) - - prevDir, err := os.Getwd() - require.NoError(t, err) - defer os.Chdir(prevDir) - os.Chdir(cwd) - - configJson, err := filepath.Abs("config.json") - require.NoError(t, err) - require.NoError(t, ioutil.WriteFile(configJson, []byte("{}"), 0600)) - - // Relative paths end up getting symlinks fully resolved. - configJsonResolved, err := filepath.EvalSymlinks(configJson) - require.NoError(t, err) - - assert.Equal(t, configJsonResolved, FindConfigFile("config.json")) - }) - - t.Run("config/config.json from various paths", func(t *testing.T) { - // Create the following directory structure: - // tmpDir1/ - // config/ - // config.json - // tmpDir2/ - // tmpDir3/ - // tmpDir4/ - // tmpDir5/ - tmpDir1, err := ioutil.TempDir("", "") - require.NoError(t, err) - defer os.RemoveAll(tmpDir1) - - err = os.Mkdir(filepath.Join(tmpDir1, "config"), 0700) - require.NoError(t, err) - - tmpDir2, err := ioutil.TempDir(tmpDir1, "") - require.NoError(t, err) - - tmpDir3, err := ioutil.TempDir(tmpDir2, "") - require.NoError(t, err) - - tmpDir4, err := ioutil.TempDir(tmpDir3, "") - require.NoError(t, err) - - tmpDir5, err := ioutil.TempDir(tmpDir4, "") - require.NoError(t, err) - - configJson := filepath.Join(tmpDir1, "config", "config.json") - require.NoError(t, ioutil.WriteFile(configJson, []byte("{}"), 0600)) - - // Relative paths end up getting symlinks fully resolved, so use this below as necessary. - configJsonResolved, err := filepath.EvalSymlinks(configJson) - require.NoError(t, err) - - testCases := []struct { - Description string - Cwd *string - FileName string - Expected string - }{ - { - "absolute path to config.json", - nil, - configJson, - configJson, - }, - { - "absolute path to config.json from directory containing config.json", - &tmpDir1, - configJson, - configJson, - }, - { - "relative path to config.json from directory containing config.json", - &tmpDir1, - "config.json", - configJsonResolved, - }, - { - "subdirectory of directory containing config.json", - &tmpDir2, - "config.json", - configJsonResolved, - }, - { - "twice-nested subdirectory of directory containing config.json", - &tmpDir3, - "config.json", - configJsonResolved, - }, - { - "thrice-nested subdirectory of directory containing config.json", - &tmpDir4, - "config.json", - configJsonResolved, - }, - { - "can't find from four nesting levels deep", - &tmpDir5, - "config.json", - "", - }, - } - - for _, testCase := range testCases { - t.Run(testCase.Description, func(t *testing.T) { - if testCase.Cwd != nil { - prevDir, err := os.Getwd() - require.NoError(t, err) - defer os.Chdir(prevDir) - os.Chdir(*testCase.Cwd) - } - - assert.Equal(t, testCase.Expected, FindConfigFile(testCase.FileName)) - }) - } - }) - - t.Run("config/config.json relative to executable", func(t *testing.T) { - osExecutable, err := os.Executable() - require.NoError(t, err) - osExecutableDir := filepath.Dir(osExecutable) - - // Force a working directory different than the executable. - cwd, err := ioutil.TempDir("", "") - require.NoError(t, err) - defer os.RemoveAll(cwd) - - prevDir, err := os.Getwd() - require.NoError(t, err) - defer os.Chdir(prevDir) - os.Chdir(cwd) - - testCases := []struct { - Description string - RelativePath string - }{ - { - "config/config.json", - ".", - }, - { - "../config/config.json", - "../", - }, - } - - for _, testCase := range testCases { - t.Run(testCase.Description, func(t *testing.T) { - // Install the config in config/config.json relative to the executable - configJson := filepath.Join(osExecutableDir, testCase.RelativePath, "config", "config.json") - require.NoError(t, os.Mkdir(filepath.Dir(configJson), 0700)) - require.NoError(t, ioutil.WriteFile(configJson, []byte("{}"), 0600)) - defer os.RemoveAll(filepath.Dir(configJson)) - - // Relative paths end up getting symlinks fully resolved. - configJsonResolved, err := filepath.EvalSymlinks(configJson) - require.NoError(t, err) - - assert.Equal(t, configJsonResolved, FindConfigFile("config.json")) - }) - } - }) -} - -func TestFindFile(t *testing.T) { - t.Run("files from various paths", func(t *testing.T) { - // Create the following directory structure: - // tmpDir1/ - // file1.json - // file2.xml - // other.txt - // tmpDir2/ - // other.txt/ [directory] - // tmpDir3/ - // tmpDir4/ - // tmpDir5/ - tmpDir1, err := ioutil.TempDir("", "") - require.NoError(t, err) - defer os.RemoveAll(tmpDir1) - - tmpDir2, err := ioutil.TempDir(tmpDir1, "") - require.NoError(t, err) - - err = os.Mkdir(filepath.Join(tmpDir2, "other.txt"), 0700) - require.NoError(t, err) - - tmpDir3, err := ioutil.TempDir(tmpDir2, "") - require.NoError(t, err) - - tmpDir4, err := ioutil.TempDir(tmpDir3, "") - require.NoError(t, err) - - tmpDir5, err := ioutil.TempDir(tmpDir4, "") - require.NoError(t, err) - - type testCase struct { - Description string - Cwd *string - FileName string - Expected string - } - - testCases := []testCase{} - - for _, fileName := range []string{"file1.json", "file2.xml", "other.txt"} { - filePath := filepath.Join(tmpDir1, fileName) - require.NoError(t, ioutil.WriteFile(filePath, []byte("{}"), 0600)) - - // Relative paths end up getting symlinks fully resolved, so use this below as necessary. - filePathResolved, err := filepath.EvalSymlinks(filePath) - require.NoError(t, err) - - testCases = append(testCases, []testCase{ - { - fmt.Sprintf("absolute path to %s", fileName), - nil, - filePath, - filePath, - }, - { - fmt.Sprintf("absolute path to %s from containing directory", fileName), - &tmpDir1, - filePath, - filePath, - }, - { - fmt.Sprintf("relative path to %s from containing directory", fileName), - &tmpDir1, - fileName, - filePathResolved, - }, - { - fmt.Sprintf("%s: subdirectory of containing directory", fileName), - &tmpDir2, - fileName, - filePathResolved, - }, - { - fmt.Sprintf("%s: twice-nested subdirectory of containing directory", fileName), - &tmpDir3, - fileName, - filePathResolved, - }, - { - fmt.Sprintf("%s: thrice-nested subdirectory of containing directory", fileName), - &tmpDir4, - fileName, - filePathResolved, - }, - { - fmt.Sprintf("%s: can't find from four nesting levels deep", fileName), - &tmpDir5, - fileName, - "", - }, - }...) - } - - for _, testCase := range testCases { - t.Run(testCase.Description, func(t *testing.T) { - if testCase.Cwd != nil { - prevDir, err := os.Getwd() - require.NoError(t, err) - defer os.Chdir(prevDir) - os.Chdir(*testCase.Cwd) - } - - assert.Equal(t, testCase.Expected, FindFile(testCase.FileName)) - }) - } - }) -} - func TestConfigFromEnviroVars(t *testing.T) { TranslationsPreInit() diff --git a/utils/fileutils/fileutils.go b/utils/fileutils/fileutils.go new file mode 100644 index 0000000000..e1eac469d4 --- /dev/null +++ b/utils/fileutils/fileutils.go @@ -0,0 +1,99 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package fileutils + +import ( + "os" + "path/filepath" +) + +var ( + commonBaseSearchPaths = []string{ + ".", + "..", + "../..", + "../../..", + } +) + +func FindPath(path string, baseSearchPaths []string, filter func(os.FileInfo) bool) string { + if filepath.IsAbs(path) { + if _, err := os.Stat(path); err == nil { + return path + } + + return "" + } + + searchPaths := []string{} + searchPaths = append(searchPaths, baseSearchPaths...) + + // Additionally attempt to search relative to the location of the running binary. + var binaryDir string + if exe, err := os.Executable(); err == nil { + if exe, err = filepath.EvalSymlinks(exe); err == nil { + if exe, err = filepath.Abs(exe); err == nil { + binaryDir = filepath.Dir(exe) + } + } + } + if binaryDir != "" { + for _, baseSearchPath := range baseSearchPaths { + searchPaths = append( + searchPaths, + filepath.Join(binaryDir, baseSearchPath), + ) + } + } + + for _, parent := range searchPaths { + found, err := filepath.Abs(filepath.Join(parent, path)) + if err != nil { + continue + } else if fileInfo, err := os.Stat(found); err == nil { + if filter != nil { + if filter(fileInfo) { + return found + } + } else { + return found + } + } + } + + return "" +} + +// FindFile looks for the given file in nearby ancestors relative to the current working +// directory as well as the directory of the executable. +func FindFile(path string) string { + return FindPath(path, commonBaseSearchPaths, func(fileInfo os.FileInfo) bool { + return !fileInfo.IsDir() + }) +} + +// fileutils.FindDir looks for the given directory in nearby ancestors relative to the current working +// directory as well as the directory of the executable, falling back to `./` if not found. +func FindDir(dir string) (string, bool) { + found := FindPath(dir, commonBaseSearchPaths, func(fileInfo os.FileInfo) bool { + return fileInfo.IsDir() + }) + if found == "" { + return "./", false + } + + return found, true +} + +// FindConfigFile attempts to find an existing configuration file. fileName can be an absolute or +// relative path or name such as "/opt/mattermost/config.json" or simply "config.json". An empty +// string is returned if no configuration is found. +func FindConfigFile(fileName string) (path string) { + found := FindFile(filepath.Join("config", fileName)) + if found == "" { + found = FindPath(fileName, []string{"."}, nil) + } + + return found +} diff --git a/utils/fileutils/fileutils_test.go b/utils/fileutils/fileutils_test.go new file mode 100644 index 0000000000..e4d66fd919 --- /dev/null +++ b/utils/fileutils/fileutils_test.go @@ -0,0 +1,293 @@ +// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. +// See License.txt for license information. + +package fileutils + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindConfigFile(t *testing.T) { + t.Run("config.json in current working directory, not inside config/", func(t *testing.T) { + // Force a unique working directory + cwd, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(cwd) + + prevDir, err := os.Getwd() + require.NoError(t, err) + defer os.Chdir(prevDir) + os.Chdir(cwd) + + configJson, err := filepath.Abs("config.json") + require.NoError(t, err) + require.NoError(t, ioutil.WriteFile(configJson, []byte("{}"), 0600)) + + // Relative paths end up getting symlinks fully resolved. + configJsonResolved, err := filepath.EvalSymlinks(configJson) + require.NoError(t, err) + + assert.Equal(t, configJsonResolved, FindConfigFile("config.json")) + }) + + t.Run("config/config.json from various paths", func(t *testing.T) { + // Create the following directory structure: + // tmpDir1/ + // config/ + // config.json + // tmpDir2/ + // tmpDir3/ + // tmpDir4/ + // tmpDir5/ + tmpDir1, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(tmpDir1) + + err = os.Mkdir(filepath.Join(tmpDir1, "config"), 0700) + require.NoError(t, err) + + tmpDir2, err := ioutil.TempDir(tmpDir1, "") + require.NoError(t, err) + + tmpDir3, err := ioutil.TempDir(tmpDir2, "") + require.NoError(t, err) + + tmpDir4, err := ioutil.TempDir(tmpDir3, "") + require.NoError(t, err) + + tmpDir5, err := ioutil.TempDir(tmpDir4, "") + require.NoError(t, err) + + configJson := filepath.Join(tmpDir1, "config", "config.json") + require.NoError(t, ioutil.WriteFile(configJson, []byte("{}"), 0600)) + + // Relative paths end up getting symlinks fully resolved, so use this below as necessary. + configJsonResolved, err := filepath.EvalSymlinks(configJson) + require.NoError(t, err) + + testCases := []struct { + Description string + Cwd *string + FileName string + Expected string + }{ + { + "absolute path to config.json", + nil, + configJson, + configJson, + }, + { + "absolute path to config.json from directory containing config.json", + &tmpDir1, + configJson, + configJson, + }, + { + "relative path to config.json from directory containing config.json", + &tmpDir1, + "config.json", + configJsonResolved, + }, + { + "subdirectory of directory containing config.json", + &tmpDir2, + "config.json", + configJsonResolved, + }, + { + "twice-nested subdirectory of directory containing config.json", + &tmpDir3, + "config.json", + configJsonResolved, + }, + { + "thrice-nested subdirectory of directory containing config.json", + &tmpDir4, + "config.json", + configJsonResolved, + }, + { + "can't find from four nesting levels deep", + &tmpDir5, + "config.json", + "", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.Description, func(t *testing.T) { + if testCase.Cwd != nil { + prevDir, err := os.Getwd() + require.NoError(t, err) + defer os.Chdir(prevDir) + os.Chdir(*testCase.Cwd) + } + + assert.Equal(t, testCase.Expected, FindConfigFile(testCase.FileName)) + }) + } + }) + + t.Run("config/config.json relative to executable", func(t *testing.T) { + osExecutable, err := os.Executable() + require.NoError(t, err) + osExecutableDir := filepath.Dir(osExecutable) + + // Force a working directory different than the executable. + cwd, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(cwd) + + prevDir, err := os.Getwd() + require.NoError(t, err) + defer os.Chdir(prevDir) + os.Chdir(cwd) + + testCases := []struct { + Description string + RelativePath string + }{ + { + "config/config.json", + ".", + }, + { + "../config/config.json", + "../", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.Description, func(t *testing.T) { + // Install the config in config/config.json relative to the executable + configJson := filepath.Join(osExecutableDir, testCase.RelativePath, "config", "config.json") + require.NoError(t, os.Mkdir(filepath.Dir(configJson), 0700)) + require.NoError(t, ioutil.WriteFile(configJson, []byte("{}"), 0600)) + defer os.RemoveAll(filepath.Dir(configJson)) + + // Relative paths end up getting symlinks fully resolved. + configJsonResolved, err := filepath.EvalSymlinks(configJson) + require.NoError(t, err) + + assert.Equal(t, configJsonResolved, FindConfigFile("config.json")) + }) + } + }) +} + +func TestFindFile(t *testing.T) { + t.Run("files from various paths", func(t *testing.T) { + // Create the following directory structure: + // tmpDir1/ + // file1.json + // file2.xml + // other.txt + // tmpDir2/ + // other.txt/ [directory] + // tmpDir3/ + // tmpDir4/ + // tmpDir5/ + tmpDir1, err := ioutil.TempDir("", "") + require.NoError(t, err) + defer os.RemoveAll(tmpDir1) + + tmpDir2, err := ioutil.TempDir(tmpDir1, "") + require.NoError(t, err) + + err = os.Mkdir(filepath.Join(tmpDir2, "other.txt"), 0700) + require.NoError(t, err) + + tmpDir3, err := ioutil.TempDir(tmpDir2, "") + require.NoError(t, err) + + tmpDir4, err := ioutil.TempDir(tmpDir3, "") + require.NoError(t, err) + + tmpDir5, err := ioutil.TempDir(tmpDir4, "") + require.NoError(t, err) + + type testCase struct { + Description string + Cwd *string + FileName string + Expected string + } + + testCases := []testCase{} + + for _, fileName := range []string{"file1.json", "file2.xml", "other.txt"} { + filePath := filepath.Join(tmpDir1, fileName) + require.NoError(t, ioutil.WriteFile(filePath, []byte("{}"), 0600)) + + // Relative paths end up getting symlinks fully resolved, so use this below as necessary. + filePathResolved, err := filepath.EvalSymlinks(filePath) + require.NoError(t, err) + + testCases = append(testCases, []testCase{ + { + fmt.Sprintf("absolute path to %s", fileName), + nil, + filePath, + filePath, + }, + { + fmt.Sprintf("absolute path to %s from containing directory", fileName), + &tmpDir1, + filePath, + filePath, + }, + { + fmt.Sprintf("relative path to %s from containing directory", fileName), + &tmpDir1, + fileName, + filePathResolved, + }, + { + fmt.Sprintf("%s: subdirectory of containing directory", fileName), + &tmpDir2, + fileName, + filePathResolved, + }, + { + fmt.Sprintf("%s: twice-nested subdirectory of containing directory", fileName), + &tmpDir3, + fileName, + filePathResolved, + }, + { + fmt.Sprintf("%s: thrice-nested subdirectory of containing directory", fileName), + &tmpDir4, + fileName, + filePathResolved, + }, + { + fmt.Sprintf("%s: can't find from four nesting levels deep", fileName), + &tmpDir5, + fileName, + "", + }, + }...) + } + + for _, testCase := range testCases { + t.Run(testCase.Description, func(t *testing.T) { + if testCase.Cwd != nil { + prevDir, err := os.Getwd() + require.NoError(t, err) + defer os.Chdir(prevDir) + os.Chdir(*testCase.Cwd) + } + + assert.Equal(t, testCase.Expected, FindFile(testCase.FileName)) + }) + } + }) +} diff --git a/utils/html.go b/utils/html.go index 8e1a788751..98cf90caae 100644 --- a/utils/html.go +++ b/utils/html.go @@ -16,6 +16,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/mattermost/mattermost-server/mlog" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/nicksnyder/go-i18n/i18n" ) @@ -26,7 +27,7 @@ type HTMLTemplateWatcher struct { } func NewHTMLTemplateWatcher(directory string) (*HTMLTemplateWatcher, error) { - templatesDir, _ := FindDir(directory) + templatesDir, _ := fileutils.FindDir(directory) mlog.Debug(fmt.Sprintf("Parsing server templates at %v", templatesDir)) ret := &HTMLTemplateWatcher{ diff --git a/utils/i18n.go b/utils/i18n.go index 4fcd7669df..76da62d05d 100644 --- a/utils/i18n.go +++ b/utils/i18n.go @@ -12,6 +12,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils/fileutils" "github.com/nicksnyder/go-i18n/i18n" ) @@ -39,7 +40,7 @@ func InitTranslations(localizationSettings model.LocalizationSettings) error { } func InitTranslationsWithDir(dir string) error { - i18nDirectory, found := FindDir(dir) + i18nDirectory, found := fileutils.FindDir(dir) if !found { return fmt.Errorf("Unable to find i18n directory") } diff --git a/utils/license.go b/utils/license.go index 95994e24e8..27365d4477 100644 --- a/utils/license.go +++ b/utils/license.go @@ -19,6 +19,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils/fileutils" ) var publicKey []byte = []byte(`-----BEGIN PUBLIC KEY----- @@ -114,7 +115,7 @@ func GetLicenseFileFromDisk(fileName string) []byte { func GetLicenseFileLocation(fileLocation string) string { if fileLocation == "" { - configDir, _ := FindDir("config") + configDir, _ := fileutils.FindDir("config") return filepath.Join(configDir, "mattermost.mattermost-license") } else { return fileLocation diff --git a/utils/license_test.go b/utils/license_test.go index c2d1b4c05f..a67d561698 100644 --- a/utils/license_test.go +++ b/utils/license_test.go @@ -5,6 +5,8 @@ package utils import ( "testing" + + "github.com/mattermost/mattermost-server/utils/fileutils" ) func TestValidateLicense(t *testing.T) { @@ -37,7 +39,7 @@ func TestGetLicenseFileFromDisk(t *testing.T) { t.Fatal("invalid bytes") } - fileBytes = GetLicenseFileFromDisk(FindConfigFile("config.json")) + fileBytes = GetLicenseFileFromDisk(fileutils.FindConfigFile("config.json")) if len(fileBytes) == 0 { // a valid bytes but should be a fail license t.Fatal("invalid bytes") } diff --git a/utils/subpath.go b/utils/subpath.go index e691aa9d8a..6c2304d43f 100644 --- a/utils/subpath.go +++ b/utils/subpath.go @@ -19,6 +19,7 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils/fileutils" ) // UpdateAssetsSubpath rewrites assets in the /client directory to assume the application is hosted @@ -28,7 +29,7 @@ func UpdateAssetsSubpath(subpath string) error { subpath = "/" } - staticDir, found := FindDir(model.CLIENT_DIR) + staticDir, found := fileutils.FindDir(model.CLIENT_DIR) if !found { return errors.New("failed to find client dir") } diff --git a/utils/testutils/testutils.go b/utils/testutils/testutils.go index 777d0aa15b..4f56c51ebd 100644 --- a/utils/testutils/testutils.go +++ b/utils/testutils/testutils.go @@ -9,11 +9,11 @@ import ( "os" "path/filepath" - "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func ReadTestFile(name string) ([]byte, error) { - path, _ := utils.FindDir("tests") + path, _ := fileutils.FindDir("tests") file, err := os.Open(filepath.Join(path, name)) if err != nil { return nil, err diff --git a/utils/timezone.go b/utils/timezone.go deleted file mode 100644 index ea5f151406..0000000000 --- a/utils/timezone.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved. -// See License.txt for license information. - -package utils - -import ( - "encoding/json" - "io/ioutil" - - "github.com/mattermost/mattermost-server/model" -) - -func LoadTimezones(fileName string) model.SupportedTimezones { - var supportedTimezones model.SupportedTimezones - - if timezoneFile := FindConfigFile(fileName); timezoneFile == "" { - return model.DefaultSupportedTimezones - } else if raw, err := ioutil.ReadFile(timezoneFile); err != nil { - return model.DefaultSupportedTimezones - } else if err := json.Unmarshal(raw, &supportedTimezones); err != nil { - return model.DefaultSupportedTimezones - } else { - return supportedTimezones - } -} diff --git a/utils/utils.go b/utils/utils.go index b156f9934d..9e35d4a885 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -11,6 +11,7 @@ import ( "strings" "github.com/mattermost/mattermost-server/model" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func StringInSlice(a string, slice []string) bool { @@ -44,7 +45,7 @@ func FileExistsInConfigFolder(filename string) bool { return false } - if _, err := os.Stat(FindConfigFile(filename)); err == nil { + if _, err := os.Stat(fileutils.FindConfigFile(filename)); err == nil { return true } return false diff --git a/web/static.go b/web/static.go index 25782ff366..98782e523e 100644 --- a/web/static.go +++ b/web/static.go @@ -16,13 +16,14 @@ import ( "github.com/mattermost/mattermost-server/mlog" "github.com/mattermost/mattermost-server/model" "github.com/mattermost/mattermost-server/utils" + "github.com/mattermost/mattermost-server/utils/fileutils" ) func (w *Web) InitStatic() { if *w.ConfigService.Config().ServiceSettings.WebserverMode != "disabled" { utils.UpdateAssetsSubpathFromConfig(w.ConfigService.Config()) - staticDir, _ := utils.FindDir(model.CLIENT_DIR) + staticDir, _ := fileutils.FindDir(model.CLIENT_DIR) mlog.Debug(fmt.Sprintf("Using client directory at %v", staticDir)) subpath, _ := utils.GetSubpathFromConfig(w.ConfigService.Config()) @@ -69,7 +70,7 @@ func root(c *Context, w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-cache, max-age=31556926, public") - staticDir, _ := utils.FindDir(model.CLIENT_DIR) + staticDir, _ := fileutils.FindDir(model.CLIENT_DIR) http.ServeFile(w, r, filepath.Join(staticDir, "root.html")) } diff --git a/web/web_test.go b/web/web_test.go index 9b42298282..5e3901af83 100644 --- a/web/web_test.go +++ b/web/web_test.go @@ -35,7 +35,7 @@ func Setup() *TestHelper { a := s.FakeApp() prevListenAddress := *a.Config().ServiceSettings.ListenAddress a.UpdateConfig(func(cfg *model.Config) { *cfg.ServiceSettings.ListenAddress = ":0" }) - serverErr := a.StartServer() + serverErr := s.Start() if serverErr != nil { panic(serverErr) }