IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

[MM-22314] Remove read limit at app/import which cause bulk import to fail (#13863)

Browse Source 
* app/import: increase buffer size to avoid failing in large inputs

* remove commented out lock

* app/import: make maxTokenSize constant

Co-authored-by: mattermod <mattermod@users.noreply.github.com>
This commit is contained in:
Ibrahim Serdar Acikgoz
2020-03-06 23:14:35 +03:00
committed by GitHub
parent e922541cc5
commit d5df6a3fc4
2 changed files with 21 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
app/import.go
View File

@@ -16,6 +16,8 @@ import (
"github.com/mattermost/mattermost-server/v5/model"
)
const maxScanTokenSize = 16 * 1024 * 1024 // Need to set a higher limit than default because some customers cross the limit. See MM-22314
func stopOnError(err LineImportWorkerError) bool {
if err.Error.Id == "api.file.upload_file.large_image.app_error" {
mlog.Warn("Large image import error", mlog.Err(err.Error))
@@ -35,6 +37,9 @@ func (a *App) bulkImportWorker(dryRun bool, wg *sync.WaitGroup, lines <-chan Lin
func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model.AppError, int) {
scanner := bufio.NewScanner(fileReader)
buf := make([]byte, 0, 64*1024)
scanner.Buffer(buf, maxScanTokenSize)
lineNumber := 0
a.Srv().Store.LockToMaster()
@@ -104,7 +109,9 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
}
// No more lines. Clear out the worker queue before continuing.
close(linesChan)
if linesChan != nil {
close(linesChan)
}
wg.Wait()
// Check no errors occurred while waiting for the queue to empty.
@@ -119,10 +126,6 @@ func (a *App) BulkImport(fileReader io.Reader, dryRun bool, workers int) (*model
return model.NewAppError("BulkImport", "app.import.bulk_import.file_scan.error", nil, err.Error(), http.StatusInternalServerError), 0
}
if err := a.finalizeImport(dryRun); err != nil {
return err, 0
}
return nil, 0
}
@@ -180,10 +183,3 @@ func (a *App) importLine(line LineImportData, dryRun bool) *model.AppError {
return model.NewAppError("BulkImport", "app.import.import_line.unknown_line_type.error", map[string]interface{}{"Type": line.Type}, "", http.StatusBadRequest)
}
}
func (a *App) finalizeImport(dryRun bool) *model.AppError {
if dryRun {
return nil
}
return nil
}

13
app/import_test.go
View File

@@ -190,6 +190,19 @@ func TestImportBulkImport(t *testing.T) {
require.NotNil(t, err, "Should have failed due to missing version line on line 1.")
require.Equal(t, 1, line, "Should have failed due to missing version line on line 1.")
// Run bulk import using a valid and large input and a \r\n line break.
t.Run("", func(t *testing.T) {
posts := `{"type": "post"` + strings.Repeat(`, "post": {"team": "`+teamName+`", "channel": "`+channelName+`", "user": "`+username+`", "message": "Repeat after me", "create_at": 193456789012}`, 1E4) + "}"
data4 := `{"type": "version", "version": 1}
{"type": "team", "team": {"type": "O", "display_name": "lskmw2d7a5ao7ppwqh5ljchvr4", "name": "` + teamName + `"}}
{"type": "channel", "channel": {"type": "O", "display_name": "xr6m6udffngark2uekvr3hoeny", "team": "` + teamName + `", "name": "` + channelName + `"}}
{"type": "user", "user": {"username": "` + username + `", "email": "` + username + `@example.com", "teams": [{"name": "` + teamName + `","theme": "` + teamTheme1 + `", "channels": [{"name": "` + channelName + `"}]}]}}
{"type": "post", "post": {"team": "` + teamName + `", "channel": "` + channelName + `", "user": "` + username + `", "message": "Hello World", "create_at": 123456789012}}`
err, line = th.App.BulkImport(strings.NewReader(data4+"\r\n"+posts), false, 2)
require.Nil(t, err, "BulkImport should have succeeded")
require.Equal(t, 0, line, "BulkImport line should be 0")
})
t.Run("First item after version without type", func(t *testing.T) {
data := `{"type": "version", "version": 1}
{"name": "custom-emoji-troll", "image": "bulkdata/emoji/trollolol.png"}`