mirror of
https://github.com/mattermost/mattermost.git
synced 2026-11-02 09:49:18 -06:00
fix: only match root-level JSONL files when importing a zip (#35481)
* fix: only match root-level JSONL files when importing a zip
When importing a Mattermost export zip, the code iterated over all files
to find the first .jsonl by extension. Exported attachments under data/
could themselves be .jsonl files, causing the import to pick an
attachment as the manifest instead of the actual root-level JSONL file.
Extract an IsRootJsonlFile helper in the imports package and use it in
the import process worker, mmctl validator, and bulk import test to
restrict the search to files with no directory component.
* test: add integration test for import with nested JSONL decoy file
Verify that BulkImportWithPath correctly picks the root-level JSONL
manifest and ignores a decoy .jsonl inside a subdirectory, covering
the fix from ad7f230f06.
This commit is contained in:
1 parent
7f0fa4937d
commit
7e7a1b582f
6 files changed
+75
-23
No files matched your search
@@ -650,7 +650,7 @@ func TestImportBulkImportWithAttachments(t *testing.T) {
|
||||
|
||||
var jsonFile io.ReadCloser
|
||||
for _, f := range importZipReader.File {
|
||||
if filepath.Ext(f.Name) != ".jsonl" {
|
||||
if !imports.IsRootJsonlFile(f.Name) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -673,6 +673,40 @@ func TestImportBulkImportWithAttachments(t *testing.T) {
|
||||
require.Len(t, files, 11)
|
||||
}
|
||||
|
||||
func TestImportBulkImportWithNestedJsonl(t *testing.T) {
|
||||
mainHelper.Parallel(t)
|
||||
th := Setup(t)
|
||||
|
||||
testsDir, _ := fileutils.FindDir("tests")
|
||||
|
||||
importFile, err := os.Open(testsDir + "/export_test_with_nested_jsonl.zip")
|
||||
require.NoError(t, err)
|
||||
defer importFile.Close()
|
||||
|
||||
info, err := importFile.Stat()
|
||||
require.NoError(t, err)
|
||||
|
||||
importZipReader, err := zip.NewReader(importFile, info.Size())
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, importZipReader)
|
||||
|
||||
var jsonFile io.ReadCloser
|
||||
for _, f := range importZipReader.File {
|
||||
if !imports.IsRootJsonlFile(f.Name) {
|
||||
continue
|
||||
}
|
||||
|
||||
jsonFile, err = f.Open()
|
||||
require.NoError(t, err)
|
||||
defer jsonFile.Close()
|
||||
break
|
||||
}
|
||||
require.NotNil(t, jsonFile)
|
||||
|
||||
_, appErr := th.App.BulkImportWithPath(th.Context, jsonFile, importZipReader, false, true, 1, model.ExportDataDir)
|
||||
require.Nil(t, appErr)
|
||||
}
|
||||
|
||||
func TestDeleteImport(t *testing.T) {
|
||||
th := Setup(t)
|
||||
|
||||
|
||||
@@ -16,6 +16,14 @@ import (
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
)
|
||||
|
||||
// IsRootJsonlFile reports whether the given zip entry name refers to a .jsonl
|
||||
// file at the root of the archive (no directory component). This is used to
|
||||
// locate the import manifest while ignoring .jsonl files that may exist as
|
||||
// exported attachments in subdirectories.
|
||||
func IsRootJsonlFile(name string) bool {
|
||||
return filepath.Ext(name) == ".jsonl" && filepath.Dir(name) == "."
|
||||
}
|
||||
|
||||
func ValidateSchemeImportData(data *SchemeImportData) *model.AppError {
|
||||
if data.Scope == nil {
|
||||
return model.NewAppError("BulkImport", "app.import.validate_scheme_import_data.null_scope.error", nil, "", http.StatusBadRequest)
|
||||
|
||||
@@ -16,6 +16,18 @@ import (
|
||||
"github.com/mattermost/mattermost/server/v8/channels/utils/fileutils"
|
||||
)
|
||||
|
||||
func TestIsRootJsonlFile(t *testing.T) {
|
||||
assert.True(t, IsRootJsonlFile("import.jsonl"))
|
||||
assert.True(t, IsRootJsonlFile("export.jsonl"))
|
||||
assert.False(t, IsRootJsonlFile("data/export.jsonl"))
|
||||
assert.False(t, IsRootJsonlFile("data/attachments/report.jsonl"))
|
||||
assert.False(t, IsRootJsonlFile("data/deep/nested/file.jsonl"))
|
||||
assert.False(t, IsRootJsonlFile("import.json"))
|
||||
assert.False(t, IsRootJsonlFile("import.zip"))
|
||||
assert.False(t, IsRootJsonlFile("data/photo.jpg"))
|
||||
assert.False(t, IsRootJsonlFile(""))
|
||||
}
|
||||
|
||||
func TestImportValidateSchemeImportData(t *testing.T) {
|
||||
// Test with minimum required valid properties and team scope.
|
||||
data := SchemeImportData{
|
||||
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"github.com/mattermost/mattermost/server/public/shared/configservice"
|
||||
"github.com/mattermost/mattermost/server/public/shared/mlog"
|
||||
"github.com/mattermost/mattermost/server/public/shared/request"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/app/imports"
|
||||
"github.com/mattermost/mattermost/server/v8/channels/jobs"
|
||||
"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
|
||||
)
|
||||
@@ -100,29 +101,28 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker {
|
||||
}
|
||||
|
||||
// find JSONL import file.
|
||||
var jsonFile io.ReadCloser
|
||||
var jsonZipFile *zip.File
|
||||
for _, f := range importZipReader.File {
|
||||
if filepath.Ext(f.Name) != ".jsonl" {
|
||||
continue
|
||||
if imports.IsRootJsonlFile(f.Name) {
|
||||
jsonZipFile = f
|
||||
break
|
||||
}
|
||||
// avoid "zip slip"
|
||||
if strings.Contains(f.Name, "..") {
|
||||
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "jsonFilePath contains path traversal", http.StatusForbidden)
|
||||
}
|
||||
|
||||
jsonFile, err = f.Open()
|
||||
if err != nil {
|
||||
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
|
||||
defer jsonFile.Close()
|
||||
break
|
||||
}
|
||||
|
||||
if jsonFile == nil {
|
||||
if jsonZipFile == nil {
|
||||
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_jsonl", nil, "jsonFile was nil", http.StatusBadRequest)
|
||||
}
|
||||
|
||||
// avoid "zip slip"
|
||||
if strings.Contains(jsonZipFile.Name, "..") {
|
||||
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "jsonFilePath contains path traversal", http.StatusForbidden)
|
||||
}
|
||||
|
||||
jsonFile, err := jsonZipFile.Open()
|
||||
if err != nil {
|
||||
return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.open_file", nil, "", http.StatusInternalServerError).Wrap(err)
|
||||
}
|
||||
defer jsonFile.Close()
|
||||
|
||||
extractContent := job.Data["extract_content"] == "true"
|
||||
// do the actual import.
|
||||
lineNumber, appErr := app.BulkImportWithPath(appContext, jsonFile, importZipReader, false, extractContent, runtime.NumCPU(), model.ExportDataDir)
|
||||
|
||||
@@ -248,12 +248,10 @@ func (v *Validator) Validate() error {
|
||||
|
||||
var jsonlZip *zip.File
|
||||
for _, zfile := range z.File {
|
||||
if filepath.Ext(zfile.Name) != ".jsonl" {
|
||||
continue
|
||||
if imports.IsRootJsonlFile(zfile.Name) {
|
||||
jsonlZip = zfile
|
||||
break
|
||||
}
|
||||
|
||||
jsonlZip = zfile
|
||||
break
|
||||
}
|
||||
if jsonlZip == nil {
|
||||
return fmt.Errorf("could not find a .jsonl file in the import archive")
|
||||
|
||||
Binary file not shown.
Reference in new issue
Block a user