diff --git a/server/channels/app/app_iface.go b/server/channels/app/app_iface.go index 0f61ffcf20..3921b66402 100644 --- a/server/channels/app/app_iface.go +++ b/server/channels/app/app_iface.go @@ -469,7 +469,7 @@ type AppIface interface { BuildSamlMetadataObject(idpMetadata []byte) (*model.SamlMetadataResponse, *model.AppError) BulkExport(ctx request.CTX, writer io.Writer, outPath string, job *model.Job, opts model.BulkExportOpts) *model.AppError BulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int) (*model.AppError, int) - BulkImportWithPath(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) + BulkImportWithPath(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun, extractContent bool, workers int, importPath string) (*model.AppError, int) CanNotifyAdmin(trial bool) bool CancelJob(c request.CTX, jobId string) *model.AppError ChannelMembersToRemove(teamID *string) ([]*model.ChannelMember, *model.AppError) @@ -590,8 +590,8 @@ type AppIface interface { DoPostActionWithCookie(c request.CTX, postID, actionId, userID, selectedOption string, cookie *model.PostActionCookie) (string, *model.AppError) DoSubscriptionRenewalCheck() DoSystemConsoleRolesCreationMigration() - DoUploadFile(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) - DoUploadFileExpectModification(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, []byte, *model.AppError) + DoUploadFile(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte, extractContent bool) (*model.FileInfo, *model.AppError) + DoUploadFileExpectModification(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte, extractContent bool) (*model.FileInfo, []byte, *model.AppError) DownloadFromURL(downloadURL string) ([]byte, error) EnableUserAccessToken(c request.CTX, token *model.UserAccessToken) *model.AppError EnvironmentConfig(filter func(reflect.StructField) bool) map[string]any diff --git a/server/channels/app/export_test.go b/server/channels/app/export_test.go index 5b0f7e6c7a..6a21b6396b 100644 --- a/server/channels/app/export_test.go +++ b/server/channels/app/export_test.go @@ -640,7 +640,7 @@ func TestBulkExport(t *testing.T) { jsonFile := extractImportFile(filepath.Join(testsDir, "import_test.zip")) defer jsonFile.Close() - appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, nil, false, 1, dir) + appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, nil, false, true, 1, dir) require.Nil(t, appErr) exportFile, err := os.Create(filepath.Join(dir, "export.zip")) @@ -661,7 +661,7 @@ func TestBulkExport(t *testing.T) { jsonFile = extractImportFile(filepath.Join(dir, "export.zip")) defer jsonFile.Close() - appErr, _ = th.App.BulkImportWithPath(th.Context, jsonFile, nil, false, 1, filepath.Join(dir, "data")) + appErr, _ = th.App.BulkImportWithPath(th.Context, jsonFile, nil, false, true, 1, filepath.Join(dir, "data")) require.Nil(t, appErr) } diff --git a/server/channels/app/file.go b/server/channels/app/file.go index c24b0e6ece..5ce7baedf2 100644 --- a/server/channels/app/file.go +++ b/server/channels/app/file.go @@ -576,7 +576,7 @@ func (a *App) UploadFileForUserAndTeam(c request.CTX, data []byte, channelID str teamId = "noteam" } - info, _, appError := a.DoUploadFileExpectModification(c, time.Now(), teamId, channelID, userId, filename, data) + info, _, appError := a.DoUploadFileExpectModification(c, time.Now(), teamId, channelID, userId, filename, data, true) if appError != nil { return nil, appError } @@ -592,8 +592,8 @@ func (a *App) UploadFileForUserAndTeam(c request.CTX, data []byte, channelID str return info, nil } -func (a *App) DoUploadFile(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { - info, _, err := a.DoUploadFileExpectModification(c, now, rawTeamId, rawChannelId, rawUserId, rawFilename, data) +func (a *App) DoUploadFile(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte, extractContent bool) (*model.FileInfo, *model.AppError) { + info, _, err := a.DoUploadFileExpectModification(c, now, rawTeamId, rawChannelId, rawUserId, rawFilename, data, extractContent) return info, err } @@ -985,7 +985,7 @@ func (t UploadFileTask) newAppError(id string, httpStatus int, extra ...any) *mo return model.NewAppError("uploadFileTask", id, params, "", httpStatus) } -func (a *App) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, []byte, *model.AppError) { +func (a *App) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte, extractContent bool) (*model.FileInfo, []byte, *model.AppError) { filename := filepath.Base(rawFilename) teamID := filepath.Base(rawTeamId) channelID := filepath.Base(rawChannelId) @@ -1063,7 +1063,10 @@ func (a *App) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTe } } - if *a.Config().FileSettings.ExtractContent { + // The extra boolean extractContent is used to turn off extraction + // during the import process. It is unnecessary overhead during the import, + // and something we can do without. + if *a.Config().FileSettings.ExtractContent && extractContent { infoCopy := *info a.Srv().GoBuffered(func() { err := a.ExtractContentFromFileInfo(c, &infoCopy) diff --git a/server/channels/app/file_bench_test.go b/server/channels/app/file_bench_test.go index a12c2ff36a..aadfc2734f 100644 --- a/server/channels/app/file_bench_test.go +++ b/server/channels/app/file_bench_test.go @@ -83,7 +83,7 @@ func BenchmarkUploadFile(b *testing.B) { title: "raw-ish DoUploadFile", f: func(b *testing.B, n int, data []byte, ext string) { info1, err := th.App.DoUploadFile(th.Context, time.Now(), teamID, channelID, - userID, fmt.Sprintf("BenchmarkDoUploadFile-%d%s", n, ext), data) + userID, fmt.Sprintf("BenchmarkDoUploadFile-%d%s", n, ext), data, true) if err != nil { b.Fatal(err) } diff --git a/server/channels/app/file_test.go b/server/channels/app/file_test.go index b89ef8c802..49517d1e1f 100644 --- a/server/channels/app/file_test.go +++ b/server/channels/app/file_test.go @@ -57,7 +57,7 @@ func TestDoUploadFile(t *testing.T) { filename := "test" data := []byte("abcd") - info1, err := th.App.DoUploadFile(th.Context, time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data) + info1, err := th.App.DoUploadFile(th.Context, time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data, true) require.Nil(t, err, "DoUploadFile should succeed with valid data") defer func() { th.App.Srv().Store().FileInfo().PermanentDelete(th.Context, info1.Id) @@ -67,7 +67,7 @@ func TestDoUploadFile(t *testing.T) { value := fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamID, channelID, userID, info1.Id, filename) assert.Equal(t, value, info1.Path, "stored file at incorrect path") - info2, err := th.App.DoUploadFile(th.Context, time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data) + info2, err := th.App.DoUploadFile(th.Context, time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data, true) require.Nil(t, err, "DoUploadFile should succeed with valid data") defer func() { th.App.Srv().Store().FileInfo().PermanentDelete(th.Context, info2.Id) @@ -77,7 +77,7 @@ func TestDoUploadFile(t *testing.T) { value = fmt.Sprintf("20070204/teams/%v/channels/%v/users/%v/%v/%v", teamID, channelID, userID, info2.Id, filename) assert.Equal(t, value, info2.Path, "stored file at incorrect path") - info3, err := th.App.DoUploadFile(th.Context, time.Date(2008, 3, 5, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data) + info3, err := th.App.DoUploadFile(th.Context, time.Date(2008, 3, 5, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data, true) require.Nil(t, err, "DoUploadFile should succeed with valid data") defer func() { th.App.Srv().Store().FileInfo().PermanentDelete(th.Context, info3.Id) @@ -87,7 +87,7 @@ func TestDoUploadFile(t *testing.T) { value = fmt.Sprintf("20080305/teams/%v/channels/%v/users/%v/%v/%v", teamID, channelID, userID, info3.Id, filename) assert.Equal(t, value, info3.Path, "stored file at incorrect path") - info4, err := th.App.DoUploadFile(th.Context, time.Date(2009, 3, 5, 1, 2, 3, 4, time.Local), "../../"+teamID, "../../"+channelID, "../../"+userID, "../../"+filename, data) + info4, err := th.App.DoUploadFile(th.Context, time.Date(2009, 3, 5, 1, 2, 3, 4, time.Local), "../../"+teamID, "../../"+channelID, "../../"+userID, "../../"+filename, data, true) require.Nil(t, err, "DoUploadFile should succeed with valid data") defer func() { th.App.Srv().Store().FileInfo().PermanentDelete(th.Context, info4.Id) @@ -97,7 +97,7 @@ func TestDoUploadFile(t *testing.T) { value = fmt.Sprintf("20090305/teams/%v/channels/%v/users/%v/%v/%v", teamID, channelID, userID, info4.Id, filename) assert.Equal(t, value, info4.Path, "stored file at incorrect path") - info5, err := th.App.DoUploadFile(th.Context, time.Date(2008, 3, 5, 1, 2, 3, 4, time.Local), teamID, channelID, model.BookmarkFileOwner, filename, data) + info5, err := th.App.DoUploadFile(th.Context, time.Date(2008, 3, 5, 1, 2, 3, 4, time.Local), teamID, channelID, model.BookmarkFileOwner, filename, data, true) require.Nil(t, err, "DoUploadFile should succeed with valid data") defer func() { th.App.Srv().Store().FileInfo().PermanentDelete(th.Context, info5.Id) @@ -355,7 +355,7 @@ func TestCopyFileInfos(t *testing.T) { filename := "test" data := []byte("abcd") - info1, err := th.App.DoUploadFile(th.Context, time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data) + info1, err := th.App.DoUploadFile(th.Context, time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data, true) require.Nil(t, err) defer func() { th.App.Srv().Store().FileInfo().PermanentDelete(th.Context, info1.Id) diff --git a/server/channels/app/import.go b/server/channels/app/import.go index ef4dd98ff7..82b9c9af78 100644 --- a/server/channels/app/import.go +++ b/server/channels/app/import.go @@ -111,7 +111,7 @@ func processAttachments(c request.CTX, line *imports.LineImportData, basePath st return nil } -func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, lines <-chan imports.LineImportWorkerData, errors chan<- imports.LineImportWorkerError) { +func (a *App) bulkImportWorker(c request.CTX, dryRun, extractContent bool, wg *sync.WaitGroup, lines <-chan imports.LineImportWorkerData, errors chan<- imports.LineImportWorkerError) { workerID := model.NewId() processedLines := uint64(0) @@ -131,7 +131,7 @@ func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, l errors <- imports.LineImportWorkerError{Error: model.NewAppError("BulkImport", "app.import.import_line.null_post.error", nil, "", http.StatusBadRequest), LineNumber: line.LineNumber} } if len(postLines) >= importMultiplePostsThreshold { - if errLine, err := a.importMultiplePostLines(c, postLines, dryRun); err != nil { + if errLine, err := a.importMultiplePostLines(c, postLines, dryRun, extractContent); err != nil { errors <- imports.LineImportWorkerError{Error: err, LineNumber: errLine} } postLines = []imports.LineImportWorkerData{} @@ -142,7 +142,7 @@ func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, l errors <- imports.LineImportWorkerError{Error: model.NewAppError("BulkImport", "app.import.import_line.null_direct_post.error", nil, "", http.StatusBadRequest), LineNumber: line.LineNumber} } if len(directPostLines) >= importMultiplePostsThreshold { - if errLine, err := a.importMultipleDirectPostLines(c, directPostLines, dryRun); err != nil { + if errLine, err := a.importMultipleDirectPostLines(c, directPostLines, dryRun, extractContent); err != nil { errors <- imports.LineImportWorkerError{Error: err, LineNumber: errLine} } directPostLines = []imports.LineImportWorkerData{} @@ -160,30 +160,30 @@ func (a *App) bulkImportWorker(c request.CTX, dryRun bool, wg *sync.WaitGroup, l } if len(postLines) > 0 { - if errLine, err := a.importMultiplePostLines(c, postLines, dryRun); err != nil { + if errLine, err := a.importMultiplePostLines(c, postLines, dryRun, extractContent); err != nil { errors <- imports.LineImportWorkerError{Error: err, LineNumber: errLine} } } if len(directPostLines) > 0 { - if errLine, err := a.importMultipleDirectPostLines(c, directPostLines, dryRun); err != nil { + if errLine, err := a.importMultipleDirectPostLines(c, directPostLines, dryRun, extractContent); err != nil { errors <- imports.LineImportWorkerError{Error: err, LineNumber: errLine} } } } func (a *App) BulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int) (*model.AppError, int) { - return a.bulkImport(c, jsonlReader, attachmentsReader, dryRun, workers, "") + return a.bulkImport(c, jsonlReader, attachmentsReader, dryRun, true, workers, "") } -func (a *App) BulkImportWithPath(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) { - return a.bulkImport(c, jsonlReader, attachmentsReader, dryRun, workers, importPath) +func (a *App) BulkImportWithPath(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun, extractContent bool, workers int, importPath string) (*model.AppError, int) { + return a.bulkImport(c, jsonlReader, attachmentsReader, dryRun, extractContent, workers, importPath) } // bulkImport will extract attachments from attachmentsReader if it is // not nil. If it is nil, it will look for attachments on the // filesystem in the locations specified by the JSONL file according // to the older behavior -func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) { +func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun, extractContent bool, workers int, importPath string) (*model.AppError, int) { scanner := bufio.NewScanner(jsonlReader) buf := make([]byte, 0, 64*1024) scanner.Buffer(buf, maxScanTokenSize) @@ -268,7 +268,7 @@ func (a *App) bulkImport(c request.CTX, jsonlReader io.Reader, attachmentsReader linesChan = make(chan imports.LineImportWorkerData, workers) for i := 0; i < workers; i++ { wg.Add(1) - go a.bulkImportWorker(c, dryRun, &wg, linesChan, errorsChan) + go a.bulkImportWorker(c, dryRun, extractContent, &wg, linesChan, errorsChan) } } diff --git a/server/channels/app/import_functions.go b/server/channels/app/import_functions.go index 039a0ee027..61e6f3b52c 100644 --- a/server/channels/app/import_functions.go +++ b/server/channels/app/import_functions.go @@ -1168,7 +1168,7 @@ func (a *App) importReaction(data *imports.ReactionImportData, post *model.Post) return nil } -func (a *App) importReplies(rctx request.CTX, data []imports.ReplyImportData, post *model.Post, teamID string) *model.AppError { +func (a *App) importReplies(rctx request.CTX, data []imports.ReplyImportData, post *model.Post, teamID string, extractContent bool) *model.AppError { var err *model.AppError usernames := []string{} for _, replyData := range data { @@ -1227,7 +1227,7 @@ func (a *App) importReplies(rctx request.CTX, data []imports.ReplyImportData, po reply.EditAt = *replyData.EditAt } - fileIDs := a.uploadAttachments(rctx, replyData.Attachments, reply, teamID) + fileIDs := a.uploadAttachments(rctx, replyData.Attachments, reply, teamID, extractContent) for _, fileID := range reply.FileIds { if _, ok := fileIDs[fileID]; !ok { a.Srv().Store().FileInfo().PermanentDelete(rctx, fileID) @@ -1272,7 +1272,7 @@ func (a *App) importReplies(rctx request.CTX, data []imports.ReplyImportData, po return nil } -func (a *App) importAttachment(rctx request.CTX, data *imports.AttachmentImportData, post *model.Post, teamID string) (*model.FileInfo, *model.AppError) { +func (a *App) importAttachment(rctx request.CTX, data *imports.AttachmentImportData, post *model.Post, teamID string, extractContent bool) (*model.FileInfo, *model.AppError) { var ( name string file io.Reader @@ -1338,7 +1338,7 @@ func (a *App) importAttachment(rctx request.CTX, data *imports.AttachmentImportD rctx.Logger().Info("Uploading file with name", mlog.String("file_name", name)) - fileInfo, appErr := a.DoUploadFile(rctx, timestamp, teamID, post.ChannelId, post.UserId, name, fileData) + fileInfo, appErr := a.DoUploadFile(rctx, timestamp, teamID, post.ChannelId, post.UserId, name, fileData, extractContent) if appErr != nil { rctx.Logger().Error("Failed to upload file", mlog.Err(appErr), mlog.String("file_name", name)) return nil, appErr @@ -1433,7 +1433,7 @@ func getPostStrID(post *model.Post) string { // importMultiplePostLines will return an error and the line that // caused it whenever possible -func (a *App) importMultiplePostLines(rctx request.CTX, lines []imports.LineImportWorkerData, dryRun bool) (int, *model.AppError) { +func (a *App) importMultiplePostLines(rctx request.CTX, lines []imports.LineImportWorkerData, dryRun, extractContent bool) (int, *model.AppError) { if len(lines) == 0 { return 0, nil } @@ -1530,7 +1530,7 @@ func (a *App) importMultiplePostLines(rctx request.CTX, lines []imports.LineImpo post.IsPinned = *line.Post.IsPinned } - fileIDs := a.uploadAttachments(rctx, line.Post.Attachments, post, team.Id) + fileIDs := a.uploadAttachments(rctx, line.Post.Attachments, post, team.Id, extractContent) for _, fileID := range post.FileIds { if _, ok := fileIDs[fileID]; !ok { a.Srv().Store().FileInfo().PermanentDelete(rctx, fileID) @@ -1618,7 +1618,7 @@ func (a *App) importMultiplePostLines(rctx request.CTX, lines []imports.LineImpo } if postWithData.postData.Replies != nil && len(*postWithData.postData.Replies) > 0 { - err := a.importReplies(rctx, *postWithData.postData.Replies, postWithData.post, postWithData.team.Id) + err := a.importReplies(rctx, *postWithData.postData.Replies, postWithData.post, postWithData.team.Id, extractContent) if err != nil { return postWithData.lineNumber, err } @@ -1629,14 +1629,14 @@ func (a *App) importMultiplePostLines(rctx request.CTX, lines []imports.LineImpo } // uploadAttachments imports new attachments and returns current attachments of the post as a map -func (a *App) uploadAttachments(rctx request.CTX, attachments *[]imports.AttachmentImportData, post *model.Post, teamID string) map[string]bool { +func (a *App) uploadAttachments(rctx request.CTX, attachments *[]imports.AttachmentImportData, post *model.Post, teamID string, extractContent bool) map[string]bool { if attachments == nil { return nil } fileIDs := make(map[string]bool) for _, attachment := range *attachments { attachment := attachment - fileInfo, err := a.importAttachment(rctx, &attachment, post, teamID) + fileInfo, err := a.importAttachment(rctx, &attachment, post, teamID, extractContent) if err != nil { if attachment.Path != nil { rctx.Logger().Warn( @@ -1742,7 +1742,7 @@ func (a *App) importDirectChannel(rctx request.CTX, data *imports.DirectChannelI // importMultipleDirectPostLines will return an error and the line // that caused it whenever possible -func (a *App) importMultipleDirectPostLines(rctx request.CTX, lines []imports.LineImportWorkerData, dryRun bool) (int, *model.AppError) { +func (a *App) importMultipleDirectPostLines(rctx request.CTX, lines []imports.LineImportWorkerData, dryRun, extractContent bool) (int, *model.AppError) { if len(lines) == 0 { return 0, nil } @@ -1843,7 +1843,7 @@ func (a *App) importMultipleDirectPostLines(rctx request.CTX, lines []imports.Li post.IsPinned = *line.DirectPost.IsPinned } - fileIDs := a.uploadAttachments(rctx, line.DirectPost.Attachments, post, "noteam") + fileIDs := a.uploadAttachments(rctx, line.DirectPost.Attachments, post, "noteam", extractContent) for _, fileID := range post.FileIds { if _, ok := fileIDs[fileID]; !ok { a.Srv().Store().FileInfo().PermanentDelete(rctx, fileID) @@ -1929,7 +1929,7 @@ func (a *App) importMultipleDirectPostLines(rctx request.CTX, lines []imports.Li } if postWithData.directPostData.Replies != nil { - if err := a.importReplies(rctx, *postWithData.directPostData.Replies, postWithData.post, "noteam"); err != nil { + if err := a.importReplies(rctx, *postWithData.directPostData.Replies, postWithData.post, "noteam", extractContent); err != nil { return postWithData.lineNumber, err } } diff --git a/server/channels/app/import_functions_test.go b/server/channels/app/import_functions_test.go index 7d3f73553e..13a465336e 100644 --- a/server/channels/app/import_functions_test.go +++ b/server/channels/app/import_functions_test.go @@ -2000,7 +2000,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 25, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, true) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, true, true) assert.NotNil(t, err) assert.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2018,7 +2018,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, true) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, true, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2035,7 +2035,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 35, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.NotNil(t, err) assert.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2053,7 +2053,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 10, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.NotNil(t, err) // Batch will fail when searching for teams, so no specific line // is associated with the error @@ -2073,7 +2073,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 7, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.NotNil(t, err) // Batch will fail when searching for channels, so no specific // line is associated with the error @@ -2093,7 +2093,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 2, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.NotNil(t, err) // Batch will fail when searching for users, so no specific line // is associated with the error @@ -2114,7 +2114,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, team.Id) @@ -2142,7 +2142,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, team.Id) @@ -2171,7 +2171,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 2, team.Id) @@ -2189,7 +2189,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 3, team.Id) @@ -2208,7 +2208,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 4, team.Id) @@ -2251,7 +2251,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err, "Expected success.") assert.Equal(t, 0, errLine) @@ -2290,7 +2290,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err, "Expected success.") assert.Equal(t, 0, errLine) @@ -2331,7 +2331,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err, "Expected success.") assert.Equal(t, 0, errLine) @@ -2377,7 +2377,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err, "Expected success.") assert.Equal(t, 0, errLine) @@ -2401,7 +2401,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err, "Expected success.") assert.Equal(t, 0, errLine) @@ -2425,7 +2425,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err, "Expected success.") assert.Equal(t, 0, errLine) @@ -2457,7 +2457,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err, "Expected success.") assert.Equal(t, 0, errLine) @@ -2520,7 +2520,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data, data2}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data, data2}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) @@ -2538,7 +2538,7 @@ func TestImportimportMultiplePostLines(t *testing.T) { }, LineNumber: 1, } - errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err = th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) @@ -2620,7 +2620,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 12, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, true) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, true, true) assert.NotNil(t, err) assert.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2639,7 +2639,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, true) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, true, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2657,7 +2657,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 2, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.NotNil(t, err) assert.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2676,7 +2676,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 7, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.NotNil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2695,7 +2695,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 8, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.NotNil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2714,7 +2714,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 9, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.NotNil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, team.Id) @@ -2733,7 +2733,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, team.Id) @@ -2762,7 +2762,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, team.Id) @@ -2792,7 +2792,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 2, team.Id) @@ -2811,7 +2811,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 3, team.Id) @@ -2830,7 +2830,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) assert.Nil(t, err) assert.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 4, team.Id) @@ -2866,7 +2866,7 @@ func TestImportImportPost(t *testing.T) { LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -2906,7 +2906,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -2946,7 +2946,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -2993,7 +2993,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -3018,7 +3018,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -3043,7 +3043,7 @@ func TestImportImportPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -3065,7 +3065,7 @@ func TestImportImportPost(t *testing.T) { LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -3097,7 +3097,7 @@ func TestImportImportPost(t *testing.T) { LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -3135,7 +3135,7 @@ func TestImportImportPost(t *testing.T) { LineNumber: 1, } - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -3368,7 +3368,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 7, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, true) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, true, true) require.NotNil(t, err) require.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") @@ -3389,7 +3389,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, true) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, true, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") @@ -3410,7 +3410,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 9, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.NotNil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") @@ -3431,7 +3431,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, "") @@ -3462,7 +3462,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, "") @@ -3493,7 +3493,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 2, "") @@ -3514,7 +3514,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 3, "") @@ -3535,7 +3535,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 4, "") @@ -3571,7 +3571,7 @@ func TestImportImportDirectPost(t *testing.T) { LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 5, "") @@ -3602,7 +3602,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 6, "") @@ -3634,7 +3634,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 7, "") @@ -3668,7 +3668,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 8, "") @@ -3726,7 +3726,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 4, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, true) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, true, true) require.NotNil(t, err) require.Equal(t, data.LineNumber, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") @@ -3748,7 +3748,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, true) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, true, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") @@ -3771,7 +3771,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 8, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.NotNil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 0, "") @@ -3793,7 +3793,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, "") @@ -3825,7 +3825,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 1, "") @@ -3857,7 +3857,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 2, "") @@ -3879,7 +3879,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 3, "") @@ -3901,7 +3901,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) AssertAllPostsCount(t, th.App, initialPostCount, 4, "") @@ -3938,7 +3938,7 @@ func TestImportImportDirectPost(t *testing.T) { LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) @@ -3977,7 +3977,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4022,7 +4022,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4074,7 +4074,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4104,7 +4104,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4137,7 +4137,7 @@ func TestImportImportDirectPost(t *testing.T) { }, LineNumber: 1, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4217,14 +4217,14 @@ func TestImportAttachment(t *testing.T) { userID := model.NewId() data := imports.AttachmentImportData{Path: &testImage} - _, err := th.App.importAttachment(th.Context, &data, &model.Post{UserId: userID, ChannelId: "some-channel"}, "some-team") + _, err := th.App.importAttachment(th.Context, &data, &model.Post{UserId: userID, ChannelId: "some-channel"}, "some-team", true) assert.Nil(t, err, "sample run without errors") attachments := GetAttachments(userID, th, t) assert.Len(t, attachments, 1) data = imports.AttachmentImportData{Path: &invalidPath} - _, err = th.App.importAttachment(th.Context, &data, &model.Post{UserId: model.NewId(), ChannelId: "some-channel"}, "some-team") + _, err = th.App.importAttachment(th.Context, &data, &model.Post{UserId: model.NewId(), ChannelId: "some-channel"}, "some-team", true) assert.NotNil(t, err, "should have failed when opening the file") assert.Equal(t, err.Id, "app.import.attachment.bad_file.error") } @@ -4319,7 +4319,7 @@ func TestImportPostAndRepliesWithAttachments(t *testing.T) { } t.Run("import with attachment", func(t *testing.T) { - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) @@ -4337,7 +4337,7 @@ func TestImportPostAndRepliesWithAttachments(t *testing.T) { t.Run("import existing post with new attachment", func(t *testing.T) { data.Post.Attachments = &[]imports.AttachmentImportData{{Path: &testImage}} - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) @@ -4374,7 +4374,7 @@ func TestImportPostAndRepliesWithAttachments(t *testing.T) { LineNumber: 7, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4435,7 +4435,7 @@ func TestImportDirectPostWithAttachments(t *testing.T) { } t.Run("Regular import of attachment", func(t *testing.T) { - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4446,7 +4446,7 @@ func TestImportDirectPostWithAttachments(t *testing.T) { }) t.Run("Attempt to import again with same file entirely, should NOT add an attachment", func(t *testing.T) { - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4471,7 +4471,7 @@ func TestImportDirectPostWithAttachments(t *testing.T) { LineNumber: 2, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportDataFake}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportDataFake}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4496,7 +4496,7 @@ func TestImportDirectPostWithAttachments(t *testing.T) { LineNumber: 2, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData2}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData2}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) @@ -4598,7 +4598,7 @@ func TestZippedImportPostAndRepliesWithAttachments(t *testing.T) { } t.Run("import with attachment", func(t *testing.T) { - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) @@ -4616,7 +4616,7 @@ func TestZippedImportPostAndRepliesWithAttachments(t *testing.T) { t.Run("import existing post with new attachment", func(t *testing.T) { data.Post.Attachments = &[]imports.AttachmentImportData{{Path: &testImage}} - errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false) + errLine, err := th.App.importMultiplePostLines(th.Context, []imports.LineImportWorkerData{data}, false, true) require.Nil(t, err) require.Equal(t, 0, errLine) @@ -4653,7 +4653,7 @@ func TestZippedImportPostAndRepliesWithAttachments(t *testing.T) { LineNumber: 7, } - errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData}, false) + errLine, err := th.App.importMultipleDirectPostLines(th.Context, []imports.LineImportWorkerData{directImportData}, false, true) require.Nil(t, err, "Expected success.") require.Equal(t, 0, errLine) diff --git a/server/channels/app/import_test.go b/server/channels/app/import_test.go index caaa287b94..603cccc0e5 100644 --- a/server/channels/app/import_test.go +++ b/server/channels/app/import_test.go @@ -459,7 +459,7 @@ func BenchmarkBulkImport(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - err, _ := th.App.BulkImportWithPath(th.Context, jsonFile, nil, false, runtime.NumCPU(), dir) + err, _ := th.App.BulkImportWithPath(th.Context, jsonFile, nil, false, true, runtime.NumCPU(), dir) require.Nil(b, err) } b.StopTimer() @@ -497,7 +497,7 @@ func TestImportBulkImportWithAttachments(t *testing.T) { th.App.UpdateConfig(func(cfg *model.Config) { cfg.TeamSettings.MaxUsersPerTeam = model.NewInt(1000) }) - appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, importZipReader, false, 1, model.ExportDataDir) + appErr, _ := th.App.BulkImportWithPath(th.Context, jsonFile, importZipReader, false, true, 1, model.ExportDataDir) require.Nil(t, appErr) adminUser, appErr := th.App.GetUserByUsername("sysadmin") diff --git a/server/channels/app/opentracing/opentracing_layer.go b/server/channels/app/opentracing/opentracing_layer.go index 16f15e476c..38e91c025a 100644 --- a/server/channels/app/opentracing/opentracing_layer.go +++ b/server/channels/app/opentracing/opentracing_layer.go @@ -1052,7 +1052,7 @@ func (a *OpenTracingAppLayer) BulkImport(c request.CTX, jsonlReader io.Reader, a return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) BulkImportWithPath(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) { +func (a *OpenTracingAppLayer) BulkImportWithPath(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, extractContent bool, workers int, importPath string) (*model.AppError, int) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.BulkImportWithPath") @@ -1064,7 +1064,7 @@ func (a *OpenTracingAppLayer) BulkImportWithPath(c request.CTX, jsonlReader io.R }() defer span.Finish() - resultVar0, resultVar1 := a.app.BulkImportWithPath(c, jsonlReader, attachmentsReader, dryRun, workers, importPath) + resultVar0, resultVar1 := a.app.BulkImportWithPath(c, jsonlReader, attachmentsReader, dryRun, extractContent, workers, importPath) if resultVar0 != nil { span.LogFields(spanlog.Error(resultVar0)) @@ -4051,7 +4051,7 @@ func (a *OpenTracingAppLayer) DoSystemConsoleRolesCreationMigration() { a.app.DoSystemConsoleRolesCreationMigration() } -func (a *OpenTracingAppLayer) DoUploadFile(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { +func (a *OpenTracingAppLayer) DoUploadFile(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte, extractContent bool) (*model.FileInfo, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DoUploadFile") @@ -4063,7 +4063,7 @@ func (a *OpenTracingAppLayer) DoUploadFile(c request.CTX, now time.Time, rawTeam }() defer span.Finish() - resultVar0, resultVar1 := a.app.DoUploadFile(c, now, rawTeamId, rawChannelId, rawUserId, rawFilename, data) + resultVar0, resultVar1 := a.app.DoUploadFile(c, now, rawTeamId, rawChannelId, rawUserId, rawFilename, data, extractContent) if resultVar1 != nil { span.LogFields(spanlog.Error(resultVar1)) @@ -4073,7 +4073,7 @@ func (a *OpenTracingAppLayer) DoUploadFile(c request.CTX, now time.Time, rawTeam return resultVar0, resultVar1 } -func (a *OpenTracingAppLayer) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, []byte, *model.AppError) { +func (a *OpenTracingAppLayer) DoUploadFileExpectModification(c request.CTX, now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte, extractContent bool) (*model.FileInfo, []byte, *model.AppError) { origCtx := a.ctx span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.DoUploadFileExpectModification") @@ -4085,7 +4085,7 @@ func (a *OpenTracingAppLayer) DoUploadFileExpectModification(c request.CTX, now }() defer span.Finish() - resultVar0, resultVar1, resultVar2 := a.app.DoUploadFileExpectModification(c, now, rawTeamId, rawChannelId, rawUserId, rawFilename, data) + resultVar0, resultVar1, resultVar2 := a.app.DoUploadFileExpectModification(c, now, rawTeamId, rawChannelId, rawUserId, rawFilename, data, extractContent) if resultVar2 != nil { span.LogFields(spanlog.Error(resultVar2)) diff --git a/server/channels/app/plugin_api_test.go b/server/channels/app/plugin_api_test.go index 49556bf584..83bb6acb0c 100644 --- a/server/channels/app/plugin_api_test.go +++ b/server/channels/app/plugin_api_test.go @@ -608,7 +608,7 @@ func TestPluginAPIGetFile(t *testing.T) { uploadTime := time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local) filename := "testGetFile" fileData := []byte("Hello World") - info, err := th.App.DoUploadFile(th.Context, uploadTime, th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, filename, fileData) + info, err := th.App.DoUploadFile(th.Context, uploadTime, th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, filename, fileData, true) require.Nil(t, err) defer func() { th.App.Srv().Store().FileInfo().PermanentDelete(th.Context, info.Id) @@ -637,6 +637,7 @@ func TestPluginAPIGetFileInfos(t *testing.T) { th.BasicUser.Id, "testFile1", []byte("testfile1 Content"), + true, ) require.Nil(t, err) defer func() { @@ -651,6 +652,7 @@ func TestPluginAPIGetFileInfos(t *testing.T) { th.BasicUser2.Id, "testFile2", []byte("testfile2 Content"), + true, ) require.Nil(t, err) defer func() { @@ -665,6 +667,7 @@ func TestPluginAPIGetFileInfos(t *testing.T) { th.BasicUser.Id, "testFile3", []byte("testfile3 Content"), + true, ) require.Nil(t, err) defer func() { diff --git a/server/channels/app/post_metadata_test.go b/server/channels/app/post_metadata_test.go index 7f005b4f2a..8fa409a018 100644 --- a/server/channels/app/post_metadata_test.go +++ b/server/channels/app/post_metadata_test.go @@ -181,7 +181,7 @@ func TestPreparePostForClient(t *testing.T) { th := setup(t) defer th.TearDown() - fileInfo, err := th.App.DoUploadFile(th.Context, time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test")) + fileInfo, err := th.App.DoUploadFile(th.Context, time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test"), true) fileInfo.Content = "test" fileInfo.ChannelId = th.BasicChannel.Id require.Nil(t, err) @@ -599,7 +599,7 @@ func TestPreparePostForClient(t *testing.T) { th := setup(t) defer th.TearDown() - fileInfo, err := th.App.DoUploadFile(th.Context, time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test")) + fileInfo, err := th.App.DoUploadFile(th.Context, time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "test.txt", []byte("test"), true) require.Nil(t, err) post, err := th.App.CreatePost(th.Context, &model.Post{ diff --git a/server/channels/app/post_test.go b/server/channels/app/post_test.go index 049e746825..c4e0ac8b82 100644 --- a/server/channels/app/post_test.go +++ b/server/channels/app/post_test.go @@ -791,7 +791,7 @@ func TestDeletePostWithFileAttachments(t *testing.T) { filename := "test" data := []byte("abcd") - info1, err := th.App.DoUploadFile(th.Context, time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data) + info1, err := th.App.DoUploadFile(th.Context, time.Date(2007, 2, 4, 1, 2, 3, 4, time.Local), teamID, channelID, userID, filename, data, true) require.Nil(t, err) defer func() { th.App.Srv().Store().FileInfo().PermanentDelete(th.Context, info1.Id) diff --git a/server/channels/app/slack.go b/server/channels/app/slack.go index ce99de0200..e958248e2d 100644 --- a/server/channels/app/slack.go +++ b/server/channels/app/slack.go @@ -34,7 +34,7 @@ func (a *App) SlackImport(c request.CTX, fileData multipart.File, fileSize int64 return a.CreateChannel(c, channel, addMember) }, DoUploadFile: func(now time.Time, rawTeamId string, rawChannelId string, rawUserId string, rawFilename string, data []byte) (*model.FileInfo, *model.AppError) { - return a.DoUploadFile(c, now, rawTeamId, rawChannelId, rawUserId, rawFilename, data) + return a.DoUploadFile(c, now, rawTeamId, rawChannelId, rawUserId, rawFilename, data, true) }, GenerateThumbnailImage: a.generateThumbnailImage, GeneratePreviewImage: a.generatePreviewImage, diff --git a/server/channels/app/user_test.go b/server/channels/app/user_test.go index 71867e4df2..b3906b7568 100644 --- a/server/channels/app/user_test.go +++ b/server/channels/app/user_test.go @@ -1060,7 +1060,7 @@ func TestPermanentDeleteUser(t *testing.T) { b := []byte("testimage") - finfo, err := th.App.DoUploadFile(th.Context, time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "testfile.txt", b) + finfo, err := th.App.DoUploadFile(th.Context, time.Now(), th.BasicTeam.Id, th.BasicChannel.Id, th.BasicUser.Id, "testfile.txt", b, true) require.Nil(t, err, "Unable to upload file. err=%v", err) diff --git a/server/channels/jobs/import_process/worker.go b/server/channels/jobs/import_process/worker.go index f7a0dfc520..90e640cf63 100644 --- a/server/channels/jobs/import_process/worker.go +++ b/server/channels/jobs/import_process/worker.go @@ -29,7 +29,7 @@ type AppIface interface { FileExists(path string) (bool, *model.AppError) FileSize(path string) (int64, *model.AppError) FileReader(path string) (filestore.ReadCloseSeeker, *model.AppError) - BulkImportWithPath(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun bool, workers int, importPath string) (*model.AppError, int) + BulkImportWithPath(c request.CTX, jsonlReader io.Reader, attachmentsReader *zip.Reader, dryRun, extractContent bool, workers int, importPath string) (*model.AppError, int) Log() *mlog.Logger } @@ -123,8 +123,9 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker { return model.NewAppError("ImportProcessWorker", "import_process.worker.do_job.missing_jsonl", nil, "jsonFile was nil", http.StatusBadRequest) } + extractContent := job.Data["extract_content"] == "true" // do the actual import. - appErr, lineNumber := app.BulkImportWithPath(appContext, jsonFile, importZipReader, false, runtime.NumCPU(), model.ExportDataDir) + appErr, lineNumber := app.BulkImportWithPath(appContext, jsonFile, importZipReader, false, extractContent, runtime.NumCPU(), model.ExportDataDir) if appErr != nil { job.Data["line_number"] = strconv.Itoa(lineNumber) return appErr diff --git a/server/cmd/mattermost/commands/import.go b/server/cmd/mattermost/commands/import.go index ab816fd62e..e02cebdc32 100644 --- a/server/cmd/mattermost/commands/import.go +++ b/server/cmd/mattermost/commands/import.go @@ -156,7 +156,7 @@ func bulkImportCmdF(command *cobra.Command, args []string) error { CommandPrettyPrintln("") - if err, lineNumber := a.BulkImportWithPath(rctx, fileReader, nil, !apply, workers, importPath); err != nil { + if err, lineNumber := a.BulkImportWithPath(rctx, fileReader, nil, true, !apply, workers, importPath); err != nil { CommandPrintErrorln(err.Error()) if lineNumber != 0 { CommandPrintErrorln(fmt.Sprintf("Error occurred on data file line %v", lineNumber)) diff --git a/server/cmd/mmctl/commands/import.go b/server/cmd/mmctl/commands/import.go index e802c82d14..f43ea52baa 100644 --- a/server/cmd/mmctl/commands/import.go +++ b/server/cmd/mmctl/commands/import.go @@ -112,6 +112,7 @@ func init() { ImportValidateCmd.Flags().Bool("check-server-duplicates", true, "Set to false to ignore teams, channels, and users already present on the server") ImportProcessCmd.Flags().Bool("bypass-upload", false, "If this is set, the file is not processed from the server, but rather directly read from the filesystem. Works only in --local mode.") + ImportProcessCmd.Flags().Bool("extract-content", true, "If this is set, document attachments will be extracted and indexed during the import process. It is advised to disable it to improve performance.") ImportListCmd.AddCommand( ImportListAvailableCmd, @@ -271,11 +272,14 @@ func importProcessCmdF(c client.Client, command *cobra.Command, args []string) e } } + extractContent, _ := command.Flags().GetBool("extract-content") + job, _, err := c.CreateJob(context.TODO(), &model.Job{ Type: model.JobTypeImportProcess, Data: map[string]string{ - "import_file": importFile, - "local_mode": strconv.FormatBool(isLocal && bypassUpload), + "import_file": importFile, + "local_mode": strconv.FormatBool(isLocal && bypassUpload), + "extract_content": strconv.FormatBool(extractContent), }, }) if err != nil { diff --git a/server/cmd/mmctl/commands/import_test.go b/server/cmd/mmctl/commands/import_test.go index 247f6f4c5e..03c2010148 100644 --- a/server/cmd/mmctl/commands/import_test.go +++ b/server/cmd/mmctl/commands/import_test.go @@ -210,7 +210,11 @@ func (s *MmctlUnitTestSuite) TestImportProcessCmdF() { importFile := "import.zip" mockJob := &model.Job{ Type: model.JobTypeImportProcess, - Data: map[string]string{"import_file": importFile, "local_mode": "false"}, + Data: map[string]string{ + "import_file": importFile, + "local_mode": "false", + "extract_content": "false", + }, } s.client. diff --git a/server/cmd/mmctl/docs/mmctl_import_process.rst b/server/cmd/mmctl/docs/mmctl_import_process.rst index 40b0d99fb5..c781f1905e 100644 --- a/server/cmd/mmctl/docs/mmctl_import_process.rst +++ b/server/cmd/mmctl/docs/mmctl_import_process.rst @@ -27,8 +27,9 @@ Options :: - --bypass-upload If this is set, the file is not processed from the server, but rather directly read from the filesystem. Works only in --local mode. - -h, --help help for process + --bypass-upload If this is set, the file is not processed from the server, but rather directly read from the filesystem. Works only in --local mode. + --extract-content If this is set, document attachments will be extracted and indexed during the import process. It is advised to disable it to improve performance. (default true) + -h, --help help for process Options inherited from parent commands ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~