IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

MM-54556: Integrate wip i18n languages (#24308)

Browse Source 
* merge languages from https://github.com/mattermost/i18n-wip

* allow only supported server locales

* Revert "merge languages from https://github.com/mattermost/i18n-wip"

This reverts commit 36de545102. We'll let
weblate populate these on start instead.

* copy fileutils to public/utils

---------

Co-authored-by: Mattermost Build <build@mattermost.com>
This commit is contained in:
Jesse Hallam 2023-09-26 12:49:27 -04:00 committed by GitHub
parent a838433263
commit 14f784a0cc
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 455 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
server/config/utils.go
View File

@ -116,14 +116,14 @@ func FixInvalidLocales(cfg *model.Config) bool {
locales := i18n.GetSupportedLocales()
if _, ok := locales[*cfg.LocalizationSettings.DefaultServerLocale]; !ok {
mlog.Warn("DefaultServerLocale must be one of the supported locales. Setting DefaultServerLocale to en as default value.", mlog.String("locale", *cfg.LocalizationSettings.DefaultServerLocale))
*cfg.LocalizationSettings.DefaultServerLocale = model.DefaultLocale
mlog.Warn("DefaultServerLocale must be one of the supported locales. Setting DefaultServerLocale to en as default value.")
changed = true
}
if _, ok := locales[*cfg.LocalizationSettings.DefaultClientLocale]; !ok {
mlog.Warn("DefaultClientLocale must be one of the supported locales. Setting DefaultClientLocale to en as default value.", mlog.String("locale", *cfg.LocalizationSettings.DefaultClientLocale))
*cfg.LocalizationSettings.DefaultClientLocale = model.DefaultLocale
mlog.Warn("DefaultClientLocale must be one of the supported locales. Setting DefaultClientLocale to en as default value.")
changed = true
}

58
server/public/shared/i18n/i18n.go
View File

@ -33,6 +33,32 @@ var T TranslateFunc
var TDefault TranslateFunc
var locales = make(map[string]string)
// supportedLocales is a hard-coded list of locales considered ready for production use. It must
// be kept in sync with ../../../../webapp/channels/src/i18n/i18n.jsx.
var supportedLocales = []string{
"de",
"en",
"en-AU",
"es",
"fr",
"it",
"hu",
"nl",
"pl",
"pt-BR",
"ro",
"sv",
"tr",
"bg",
"ru",
"uk",
"fa",
"ko",
"zh-CN",
"zh-TW",
"ja",
}
var defaultServerLocale string
var defaultClientLocale string
@ -67,7 +93,13 @@ func initTranslationsWithDir(dir string) error {
for _, f := range files {
if filepath.Ext(f.Name()) == ".json" {
filename := f.Name()
locales[strings.Split(filename, ".")[0]] = filepath.Join(dir, filename)
locale := strings.Split(filename, ".")[0]
if !isSupportedLocale(locale) {
continue
}
locales[locale] = filepath.Join(dir, filename)
if err := i18n.LoadTranslationFile(filepath.Join(dir, filename)); err != nil {
return err
@ -89,8 +121,13 @@ func GetTranslationFuncForDir(dir string) (TranslationFuncByLocal, error) {
continue
}
locale := strings.Split(f.Name(), ".")[0]
if !isSupportedLocale(locale) {
continue
}
filename := f.Name()
availableLocals[strings.Split(filename, ".")[0]] = filepath.Join(dir, filename)
availableLocals[locale] = filepath.Join(dir, filename)
if err := bundle.LoadTranslationFile(filepath.Join(dir, filename)); err != nil {
return nil, err
}
@ -116,7 +153,12 @@ func GetTranslationFuncForDir(dir string) (TranslationFuncByLocal, error) {
func getTranslationsBySystemLocale() (TranslateFunc, error) {
locale := defaultServerLocale
if _, ok := locales[locale]; !ok {
mlog.Warn("Failed to load system translations for", mlog.String("locale", locale), mlog.String("attempting to fall back to default locale", defaultLocale))
mlog.Warn("Failed to load system translations for selected locale, attempting to fall back to default", mlog.String("locale", locale), mlog.String("default_locale", defaultLocale))
locale = defaultLocale
}
if !isSupportedLocale(locale) {
mlog.Warn("Selected locale is unsupported, attempting to fall back to default", mlog.String("locale", locale), mlog.String("default_locale", defaultLocale))
locale = defaultLocale
}
@ -222,3 +264,13 @@ func IdentityTfunc() TranslateFunc {
return translationID
}
}
func isSupportedLocale(locale string) bool {
for _, supportedLocale := range supportedLocales {
if locale == supportedLocale {
return true
}
}
return false
}

157
server/public/shared/i18n/i18n_test.go
View File

@ -4,12 +4,17 @@
package i18n
import (
"fmt"
"os"
"path/filepath"
"testing"
"github.com/mattermost/go-i18n/i18n/bundle"
"github.com/mattermost/go-i18n/i18n/language"
"github.com/mattermost/go-i18n/i18n/translation"
"github.com/mattermost/mattermost/server/public/utils"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
var htmlTestTranslationBundle *bundle.Bundle
@ -67,3 +72,155 @@ func TestEscapeForHTML(t *testing.T) {
})
}
}
func TestInitTranslationsWithDir(t *testing.T) {
i18nDir, found := utils.FindDir("server/i18n")
require.True(t, found, "unable to find i18n dir")
setup := func(t *testing.T, localesToCopy map[string]string) string {
tempDir, err := os.MkdirTemp(os.TempDir(), "TestGetTranslationFuncForDir")
require.NoError(t, err, "unable to create temporary directory")
t.Cleanup(func() {
err = os.RemoveAll(tempDir)
require.NoError(t, err)
})
for locale, fromLocale := range localesToCopy {
err = utils.CopyFile(
filepath.Join(i18nDir, fmt.Sprintf("%s.json", fromLocale)),
filepath.Join(tempDir, fmt.Sprintf("%s.json", locale)),
)
require.NoError(t, err)
}
return tempDir
}
t.Run("unsupported locale ignored", func(t *testing.T) {
tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
err := initTranslationsWithDir(tempDir)
require.NoError(t, err)
_, found := locales["zz"]
require.False(t, found, "should have ignored unsupported locale")
})
t.Run("malformed, unsupported locale ignored", func(t *testing.T) {
tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
err := os.WriteFile(filepath.Join(tempDir, "xx.json"), []byte{'{'}, os.ModePerm)
require.NoError(t, err)
err = initTranslationsWithDir(tempDir)
require.NoError(t, err)
_, found := locales["xx"]
require.False(t, found, "should have ignored malformed, unsupported locale")
})
t.Run("malformed, supported locale causes error", func(t *testing.T) {
tempDir := setup(t, map[string]string{"fr": "fr", "zz": "en"})
err := os.WriteFile(filepath.Join(tempDir, "en.json"), []byte{'{'}, os.ModePerm)
require.NoError(t, err)
err = initTranslationsWithDir(tempDir)
require.Error(t, err, "should have failed to load malformed, supported locale")
})
t.Run("known locales loaded ", func(t *testing.T) {
tempDir := setup(t, map[string]string{"en": "en", "fr": "fr"})
err := initTranslationsWithDir(tempDir)
require.NoError(t, err)
_, found := locales["en"]
require.True(t, found, "should have found en locale")
_, found = locales["fr"]
require.True(t, found, "should have found fr locale")
_, found = locales["es"]
require.False(t, found, "should not have found unloaded es locale")
})
}
func TestGetTranslationFuncForDir(t *testing.T) {
i18nDir, found := utils.FindDir("server/i18n")
require.True(t, found, "unable to find i18n dir")
setup := func(t *testing.T, localesToCopy map[string]string) string {
tempDir, err := os.MkdirTemp(os.TempDir(), "TestGetTranslationFuncForDir")
require.NoError(t, err, "unable to create temporary directory")
t.Cleanup(func() {
err = os.RemoveAll(tempDir)
require.NoError(t, err)
})
for locale, fromLocale := range localesToCopy {
err = utils.CopyFile(
filepath.Join(i18nDir, fmt.Sprintf("%s.json", fromLocale)),
filepath.Join(tempDir, fmt.Sprintf("%s.json", locale)),
)
require.NoError(t, err)
}
return tempDir
}
t.Run("unknown locale falls back to english", func(t *testing.T) {
tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
translationFunc, err := GetTranslationFuncForDir(tempDir)
require.NoError(t, err)
require.NotNil(t, translationFunc)
require.Equal(t, "December", translationFunc("unknown")("December"))
})
t.Run("unsupported locale falls back to english", func(t *testing.T) {
tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
translationFunc, err := GetTranslationFuncForDir(tempDir)
require.NoError(t, err)
require.NotNil(t, translationFunc)
require.Equal(t, "December", translationFunc("zz")("December"))
})
t.Run("malformed, unsupported locale ignored and falls back to english", func(t *testing.T) {
tempDir := setup(t, map[string]string{"en": "en", "fr": "fr", "zz": "en"})
err := os.WriteFile(filepath.Join(tempDir, "xx.json"), []byte{'{'}, os.ModePerm)
require.NoError(t, err)
translationFunc, err := GetTranslationFuncForDir(tempDir)
require.NoError(t, err)
require.NotNil(t, translationFunc)
require.Equal(t, "December", translationFunc("xx")("December"))
})
t.Run("malformed, supported locale causes error", func(t *testing.T) {
tempDir := setup(t, map[string]string{"fr": "fr", "zz": "en"})
err := os.WriteFile(filepath.Join(tempDir, "en.json"), []byte{'{'}, os.ModePerm)
require.NoError(t, err)
translationFunc, err := GetTranslationFuncForDir(tempDir)
require.Error(t, err)
require.Nil(t, translationFunc)
})
t.Run("known locale matches", func(t *testing.T) {
tempDir := setup(t, map[string]string{"en": "en", "fr": "fr"})
translationFunc, err := GetTranslationFuncForDir(tempDir)
require.NoError(t, err)
require.NotNil(t, translationFunc)
require.Equal(t, "Décembre", translationFunc("fr")("December"))
require.Equal(t, "December", translationFunc("en")("December"))
})
}

118
server/public/utils/fileutils.go Normal file
View File

@ -0,0 +1,118 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package utils
import (
"os"
"path/filepath"
)
func CommonBaseSearchPaths() []string {
paths := []string{
".",
"..",
"../..",
"../../..",
"../../../..",
}
// this enables the server to be used in tests from a different repository
if mmPath := os.Getenv("MM_SERVER_PATH"); mmPath != "" {
paths = append(paths, mmPath)
}
return paths
}
func findPath(path string, baseSearchPaths []string, workingDirFirst bool, filter func(os.FileInfo) bool) string {
if filepath.IsAbs(path) {
if _, err := os.Stat(path); err == nil {
return path
}
return ""
}
searchPaths := []string{}
if workingDirFirst {
searchPaths = append(searchPaths, baseSearchPaths...)
}
// Attempt to search relative to the location of the running binary either before
// or after searching relative to the working directory, depending on `workingDirFirst`.
var binaryDir string
if exe, err := os.Executable(); err == nil {
if exe, err = filepath.EvalSymlinks(exe); err == nil {
if exe, err = filepath.Abs(exe); err == nil {
binaryDir = filepath.Dir(exe)
}
}
}
if binaryDir != "" {
for _, baseSearchPath := range baseSearchPaths {
searchPaths = append(
searchPaths,
filepath.Join(binaryDir, baseSearchPath),
)
}
}
if !workingDirFirst {
searchPaths = append(searchPaths, baseSearchPaths...)
}
for _, parent := range searchPaths {
found, err := filepath.Abs(filepath.Join(parent, path))
if err != nil {
continue
} else if fileInfo, err := os.Stat(found); err == nil {
if filter != nil {
if filter(fileInfo) {
return found
}
} else {
return found
}
}
}
return ""
}
func FindPath(path string, baseSearchPaths []string, filter func(os.FileInfo) bool) string {
return findPath(path, baseSearchPaths, true, filter)
}
// FindFile looks for the given file in nearby ancestors relative to the current working
// directory as well as the directory of the executable.
func FindFile(path string) string {
return FindPath(path, CommonBaseSearchPaths(), func(fileInfo os.FileInfo) bool {
return !fileInfo.IsDir()
})
}
// fileutils.FindDir looks for the given directory in nearby ancestors relative to the current working
// directory as well as the directory of the executable, falling back to `./` if not found.
func FindDir(dir string) (string, bool) {
found := FindPath(dir, CommonBaseSearchPaths(), func(fileInfo os.FileInfo) bool {
return fileInfo.IsDir()
})
if found == "" {
return "./", false
}
return found, true
}
// FindDirRelBinary looks for the given directory in nearby ancestors relative to the
// directory of the executable, then relative to the working directory, falling back to `./` if not found.
func FindDirRelBinary(dir string) (string, bool) {
found := findPath(dir, CommonBaseSearchPaths(), false, func(fileInfo os.FileInfo) bool {
return fileInfo.IsDir()
})
if found == "" {
return "./", false
}
return found, true
}

123
server/public/utils/fileutils_test.go Normal file
View File

@ -0,0 +1,123 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package utils
import (
"fmt"
"os"
"path/filepath"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestFindFile(t *testing.T) {
t.Run("files from various paths", func(t *testing.T) {
// Create the following directory structure:
// tmpDir1/
// file1.json
// file2.xml
// other.txt
// tmpDir2/
// other.txt/ [directory]
// tmpDir3/
// tmpDir4/
// tmpDir5/
tmpDir1, err := os.MkdirTemp("", "")
require.NoError(t, err)
defer os.RemoveAll(tmpDir1)
tmpDir2, err := os.MkdirTemp(tmpDir1, "")
require.NoError(t, err)
err = os.Mkdir(filepath.Join(tmpDir2, "other.txt"), 0700)
require.NoError(t, err)
tmpDir3, err := os.MkdirTemp(tmpDir2, "")
require.NoError(t, err)
tmpDir4, err := os.MkdirTemp(tmpDir3, "")
require.NoError(t, err)
tmpDir5, err := os.MkdirTemp(tmpDir4, "")
require.NoError(t, err)
type testCase struct {
Description string
Cwd *string
FileName string
Expected string
}
testCases := []testCase{}
for _, fileName := range []string{"file1.json", "file2.xml", "other.txt"} {
filePath := filepath.Join(tmpDir1, fileName)
require.NoError(t, os.WriteFile(filePath, []byte("{}"), 0600))
// Relative paths end up getting symlinks fully resolved, so use this below as necessary.
filePathResolved, err := filepath.EvalSymlinks(filePath)
require.NoError(t, err)
testCases = append(testCases, []testCase{
{
fmt.Sprintf("absolute path to %s", fileName),
nil,
filePath,
filePath,
},
{
fmt.Sprintf("absolute path to %s from containing directory", fileName),
&tmpDir1,
filePath,
filePath,
},
{
fmt.Sprintf("relative path to %s from containing directory", fileName),
&tmpDir1,
fileName,
filePathResolved,
},
{
fmt.Sprintf("%s: subdirectory of containing directory", fileName),
&tmpDir2,
fileName,
filePathResolved,
},
{
fmt.Sprintf("%s: twice-nested subdirectory of containing directory", fileName),
&tmpDir3,
fileName,
filePathResolved,
},
{
fmt.Sprintf("%s: thrice-nested subdirectory of containing directory", fileName),
&tmpDir4,
fileName,
filePathResolved,
},
{
fmt.Sprintf("%s: quadruple-nested subdirectory of containing directory", fileName),
&tmpDir5,
fileName,
filePathResolved,
},
}...)
}
for _, testCase := range testCases {
t.Run(testCase.Description, func(t *testing.T) {
if testCase.Cwd != nil {
prevDir, err := os.Getwd()
require.NoError(t, err)
defer os.Chdir(prevDir)
os.Chdir(*testCase.Cwd)
}
assert.Equal(t, testCase.Expected, FindFile(testCase.FileName))
})
}
})
}