IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2026-08-26 21:27:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fix import failures for Japanese filenames with dakuten on macOS (#35204)

Browse Source 
* 🐛 fix: normalize Unicode filenames in import attachment lookup

Fix import failures for files with Japanese dakuten/handakuten characters
(e.g., ガ, パ, べ) on macOS.

macOS stores filenames in NFD (decomposed) form while Linux/Windows use
NFC (composed) form. This mismatch caused attachment lookup failures
when zip filenames and JSONL paths used different normalization forms.

Changes:
- Add NormalizeFilename utility function using golang.org/x/text/unicode/norm
- Normalize filenames when building attachment maps from zip files
- Normalize paths when looking up attachments in maps
- Apply fixes to both server (import.go) and mmctl (validate.go)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

* avoid duplicating normalizeFilename

* add coverage for Korean filenames

---------

Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Co-authored-by: Jesse Hallam <jesse@mattermost.com>
Co-authored-by: Mattermost Build <build@mattermost.com>
This commit is contained in:
135yshr
2026-03-18 12:16:55 +00:00
committed by GitHub
co-authored by Claude Opus 4.5 Jesse Hallam Mattermost Build
parent 04baff8c8d
commit 314ed3756a
4 changed files with 159 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/channels/app/import.go
+10 -5
View File
@@ -19,6 +19,7 @@ import (
"github.com/mattermost/mattermost/server/public/shared/mlog"
"github.com/mattermost/mattermost/server/public/shared/request"
"github.com/mattermost/mattermost/server/v8/channels/app/imports"
"github.com/mattermost/mattermost/server/v8/channels/utils"
)
type ReactionImportData = imports.ReactionImportData // part of the app interface
@@ -63,7 +64,8 @@ func processAttachmentPaths(rctx request.CTX, files *[]imports.AttachmentImportD
}
if len(filesMap) > 0 {
if (*files)[i].Data, ok = filesMap[*f.Path]; !ok {
normalizedPath := utils.NormalizeFilename(*f.Path)
if (*files)[i].Data, ok = filesMap[normalizedPath]; !ok {
errs = append(errs, fmt.Errorf("attachment %q not found in map", originalPath))
continue
}
@@ -108,7 +110,8 @@ func processAttachments(rctx request.CTX, line *imports.LineImportData, basePath
*line.User.ProfileImage = path
if len(filesMap) > 0 {
if line.User.ProfileImageData, ok = filesMap[path]; !ok {
normalizedPath := utils.NormalizeFilename(path)
if line.User.ProfileImageData, ok = filesMap[normalizedPath]; !ok {
return fmt.Errorf("attachment %q not found in map", path)
}
}
@@ -122,7 +125,8 @@ func processAttachments(rctx request.CTX, line *imports.LineImportData, basePath
*line.Bot.ProfileImage = path
if len(filesMap) > 0 {
if line.Bot.ProfileImageData, ok = filesMap[path]; !ok {
normalizedPath := utils.NormalizeFilename(path)
if line.Bot.ProfileImageData, ok = filesMap[normalizedPath]; !ok {
return fmt.Errorf("attachment %q not found in map", path)
}
}
@@ -136,7 +140,8 @@ func processAttachments(rctx request.CTX, line *imports.LineImportData, basePath
*line.Emoji.Image = path
if len(filesMap) > 0 {
if line.Emoji.Data, ok = filesMap[path]; !ok {
normalizedPath := utils.NormalizeFilename(path)
if line.Emoji.Data, ok = filesMap[normalizedPath]; !ok {
return fmt.Errorf("attachment %q not found in map", path)
}
}
@@ -237,7 +242,7 @@ func (a *App) bulkImport(rctx request.CTX, jsonlReader io.Reader, attachmentsRea
if attachmentsReader != nil {
attachedFiles = make(map[string]*zip.File, len(attachmentsReader.File))
for _, fi := range attachmentsReader.File {
attachedFiles[fi.Name] = fi
attachedFiles[utils.NormalizeFilename(fi.Name)] = fi
}
}
server/channels/utils/unicode.go
+15
View File
@@ -0,0 +1,15 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package utils
import "golang.org/x/text/unicode/norm"
// NormalizeFilename normalizes a filename to NFC (composed) form.
// This ensures consistent string comparison between filesystems that use
// different Unicode normalization forms (macOS uses NFD, Linux/Windows use NFC).
// This is particularly important for Japanese dakuten/handakuten characters
// (e.g., "ガ" can be represented as U+30AC (NFC) or U+30AB + U+3099 (NFD)).
func NormalizeFilename(name string) string {
return norm.NFC.String(name)
}
server/channels/utils/unicode_test.go
+124
View File
@@ -0,0 +1,124 @@
// Copyright (c) 2015-present Mattermost, Inc. All Rights Reserved.
// See LICENSE.txt for license information.
package utils
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNormalizeFilename(t *testing.T) {
tests := []struct {
name string
input string
expected string
}{
{
name: "ASCII only",
input: "test.jpg",
expected: "test.jpg",
},
{
name: "Japanese katakana dakuten NFC",
input: "\u30AC", // ガ (NFC)
expected: "\u30AC",
},
{
name: "Japanese katakana dakuten NFD",
input: "\u30AB\u3099", // カ + combining dakuten → ガ
expected: "\u30AC",
},
{
name: "Japanese katakana handakuten NFC",
input: "\u30D1", // パ (NFC)
expected: "\u30D1",
},
{
name: "Japanese katakana handakuten NFD",
input: "\u30CF\u309A", // ハ + combining handakuten → パ
expected: "\u30D1",
},
{
name: "Japanese hiragana dakuten NFC",
input: "\u3079", // べ (NFC)
expected: "\u3079",
},
{
name: "Japanese hiragana dakuten NFD",
input: "\u3078\u3099", // へ + combining dakuten → べ
expected: "\u3079",
},
{
name: "Mixed path with NFD",
input: "data/\u30AB\u3099test.jpg", // data/カ゛test.jpg
expected: "data/\u30ACtest.jpg", // data/ガtest.jpg
},
{
name: "Complex Japanese filename NFD",
input: "\u304B\u3099\u304D\u3099\u3050", // が + ぎ + ぐ (NFD: か゛き゛く゛)
expected: "\u304C\u304E\u3050", // がぎぐ (NFC)
},
{
name: "Path with multiple NFD characters",
input: "data/\u30D5\u309A\u30ED\u30B7\u3099\u30A7\u30AF\u30C8.png", // data/プロジェクト.png (NFD)
expected: "data/\u30D7\u30ED\u30B8\u30A7\u30AF\u30C8.png", // data/プロジェクト.png (NFC)
},
{
name: "Empty string",
input: "",
expected: "",
},
{
name: "Already NFC normalized",
input: "ファイル名.txt",
expected: "ファイル名.txt",
},
{
name: "Korean Hangul NFC",
input: "\uAC00", // 가 (NFC precomposed)
expected: "\uAC00",
},
{
name: "Korean Hangul NFD",
input: "\u1100\u1161", // ᄀ + ᅡ (NFD Jamo) → 가
expected: "\uAC00",
},
{
name: "Korean word NFD",
input: "\u1112\u1161\u11AB\u1100\u1173\u11AF", // 한글 (NFD Jamo: 한글)
expected: "\uD55C\uAE00", // 한글 (NFC)
},
{
name: "Korean filename with path NFD",
input: "data/\u1111\u1161\u110B\u1175\u11AF.txt", // 파일 (NFD) + .txt
expected: "data/\uD30C\uC77C.txt", // 파일.txt (NFC)
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := NormalizeFilename(tt.input)
assert.Equal(t, tt.expected, result)
})
}
}
func TestNormalizeFilenameIdempotent(t *testing.T) {
// NFC normalization should be idempotent
inputs := []string{
"test.jpg",
"\u30AC", // ガ (NFC)
"\u30AB\u3099", // カ + combining dakuten (NFD)
"data/テスト.jpg",
"\uD55C\uAE00", // 한글 (NFC)
"\u1112\u1161\u11AB\u1100\u1173\u11AF", // 한글 (NFD Jamo)
"",
}
for _, input := range inputs {
first := NormalizeFilename(input)
second := NormalizeFilename(first)
assert.Equal(t, first, second, "NormalizeFilename should be idempotent for input: %q", input)
}
}
server/cmd/mmctl/commands/importer/validate.go
+10 -8
View File
@@ -27,6 +27,7 @@ import (
"github.com/mattermost/mattermost/server/public/model"
"github.com/mattermost/mattermost/server/v8/channels/app/imports"
"github.com/mattermost/mattermost/server/v8/channels/utils"
_ "golang.org/x/image/webp" // image decoder
"github.com/mattermost/mattermost/server/v8/cmd/mmctl/printer"
@@ -262,10 +263,11 @@ func (v *Validator) Validate() error {
if zfile.FileInfo().IsDir() {
continue
}
if strings.HasPrefix(zfile.Name, "data/") {
v.attachments[zfile.Name] = zfile
normalizedName := utils.NormalizeFilename(zfile.Name)
if strings.HasPrefix(normalizedName, "data/") {
v.attachments[normalizedName] = zfile
}
v.allFileNames = append(v.allFileNames, zfile.Name)
v.allFileNames = append(v.allFileNames, normalizedName)
}
}
@@ -813,9 +815,9 @@ func (v *Validator) validatePost(info ImportFileInfo, line imports.LineImportDat
continue
}
attachmentPath := *attachment.Path
attachmentPath := utils.NormalizeFilename(*attachment.Path)
if _, ok := v.attachments[attachmentPath]; !ok {
attachmentPath = path.Join("data", *attachment.Path)
attachmentPath = utils.NormalizeFilename(path.Join("data", *attachment.Path))
}
if _, ok := v.attachments[attachmentPath]; !ok {
@@ -950,9 +952,9 @@ func (v *Validator) validateDirectPost(info ImportFileInfo, line imports.LineImp
continue
}
attachmentPath := *attachment.Path
attachmentPath := utils.NormalizeFilename(*attachment.Path)
if _, ok := v.attachments[attachmentPath]; !ok {
attachmentPath = path.Join("data", *attachment.Path)
attachmentPath = utils.NormalizeFilename(path.Join("data", *attachment.Path))
}
if _, ok := v.attachments[attachmentPath]; !ok {
@@ -1003,7 +1005,7 @@ func (v *Validator) validateEmoji(info ImportFileInfo, line imports.LineImportDa
}
if !v.ignoreAttachments && data.Image != nil {
attachmentPath := path.Join("data", *data.Image)
attachmentPath := utils.NormalizeFilename(path.Join("data", *data.Image))
zfile, ok := v.attachments[attachmentPath]
if !ok {