diff --git a/api4/job.go b/api4/job.go index 8d83d936b6..34191febd8 100644 --- a/api4/job.go +++ b/api4/job.go @@ -4,9 +4,13 @@ package api4 import ( + "fmt" "net/http" + "strconv" + "time" "github.com/mattermost/mattermost-server/v5/audit" + "github.com/mattermost/mattermost-server/v5/mlog" "github.com/mattermost/mattermost-server/v5/model" ) @@ -14,6 +18,7 @@ func (api *API) InitJob() { api.BaseRoutes.Jobs.Handle("", api.ApiSessionRequired(getJobs)).Methods("GET") api.BaseRoutes.Jobs.Handle("", api.ApiSessionRequired(createJob)).Methods("POST") api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}", api.ApiSessionRequired(getJob)).Methods("GET") + api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}/download", api.ApiSessionRequiredTrustRequester(downloadJob)).Methods("GET") api.BaseRoutes.Jobs.Handle("/{job_id:[A-Za-z0-9]+}/cancel", api.ApiSessionRequired(cancelJob)).Methods("POST") api.BaseRoutes.Jobs.Handle("/type/{job_type:[A-Za-z0-9_-]+}", api.ApiSessionRequired(getJobsByType)).Methods("GET") } @@ -38,6 +43,68 @@ func getJob(c *Context, w http.ResponseWriter, r *http.Request) { w.Write([]byte(job.ToJson())) } +func downloadJob(c *Context, w http.ResponseWriter, r *http.Request) { + config := c.App.Config() + const FILE_PATH = "export/%s/%s" + fileInfo := map[string]map[string]string{ + "csv": { + "fileName": "csv_export.zip", + "fileMime": "application/zip", + }, + "actiance": { + "fileName": "actiance_export.zip", + "fileMime": "application/zip", + }, + } + + c.RequireJobId() + if c.Err != nil { + return + } + + if !*config.MessageExportSettings.DownloadExportResults { + c.Err = model.NewAppError("downloadExportResultsNotEnabled", "app.job.download_export_results_not_enabled", nil, "", http.StatusNotImplemented) + return + } + + if !c.App.SessionHasPermissionTo(*c.App.Session(), model.PERMISSION_MANAGE_JOBS) { + c.SetPermissionError(model.PERMISSION_MANAGE_JOBS) + return + } + + job, err := c.App.GetJob(c.Params.JobId) + if err != nil { + mlog.Error(err.Error()) + c.Err = err + return + } + + isDownloadable, _ := strconv.ParseBool(job.Data["is_downloadable"]) + if !isDownloadable { + c.Err = model.NewAppError("unableToDownloadJob", "api.job.unable_to_download_job", nil, "", http.StatusBadRequest) + return + } + + filePath := fmt.Sprintf(FILE_PATH, job.Id, fileInfo[job.Data["export_type"]]["fileName"]) + fileReader, err := c.App.FileReader(filePath) + if err != nil { + mlog.Error(err.Error()) + c.Err = err + c.Err.StatusCode = http.StatusNotFound + return + } + defer fileReader.Close() + + // We are able to pass 0 for content size due to the fact that Golang's serveContent (https://golang.org/src/net/http/fs.go) + // already sets that for us + err = writeFileResponse(fileInfo[job.Data["export_type"]]["fileName"], fileInfo[job.Data["export_type"]]["fileMime"], 0, time.Unix(0, job.LastActivityAt*int64(1000*1000)), *c.App.Config().ServiceSettings.WebserverMode, fileReader, true, w, r) + if err != nil { + mlog.Error(err.Error()) + c.Err = err + return + } +} + func createJob(c *Context, w http.ResponseWriter, r *http.Request) { job := model.JobFromJson(r.Body) if job == nil { diff --git a/api4/job_test.go b/api4/job_test.go index 80275c5158..de846f29a5 100644 --- a/api4/job_test.go +++ b/api4/job_test.go @@ -4,6 +4,8 @@ package api4 import ( + "os" + "path/filepath" "strings" "testing" @@ -172,6 +174,68 @@ func TestGetJobsByType(t *testing.T) { CheckForbiddenStatus(t, resp) } +func TestDownloadJob(t *testing.T) { + th := Setup(t).InitBasic() + defer th.TearDown() + jobName := model.NewId() + job := &model.Job{ + Id: jobName, + Type: model.JOB_TYPE_MESSAGE_EXPORT, + Data: map[string]string{ + "export_type": "csv", + }, + Status: model.JOB_STATUS_SUCCESS, + } + + // DownloadExportResults is not set to true so we should get a not implemented error status + _, resp := th.Client.DownloadJob(job.Id) + CheckNotImplementedStatus(t, resp) + + th.App.UpdateConfig(func(cfg *model.Config) { + *cfg.MessageExportSettings.DownloadExportResults = true + }) + + // Normal user cannot download the results of these job (Doesn't have permission) + _, resp = th.Client.DownloadJob(job.Id) + CheckForbiddenStatus(t, resp) + + // System admin trying to download the results of a non-existant job + _, resp = th.SystemAdminClient.DownloadJob(job.Id) + CheckNotFoundStatus(t, resp) + + // Here we have a job that exist in our database but the results do not exist therefore when we try to download the results + // as a system admin, we should get a not found status. + _, err := th.App.Srv().Store.Job().Save(job) + require.Nil(t, err) + defer th.App.Srv().Store.Job().Delete(job.Id) + + filePath := "./data/export/" + job.Id + "/testdat.txt" + mkdirAllErr := os.MkdirAll(filepath.Dir(filePath), 0770) + require.Nil(t, mkdirAllErr) + os.Create(filePath) + + _, resp = th.SystemAdminClient.DownloadJob(job.Id) + CheckBadRequestStatus(t, resp) + + job.Data["is_downloadable"] = "true" + updateStatus, err := th.App.Srv().Store.Job().UpdateOptimistically(job, model.JOB_STATUS_SUCCESS) + require.True(t, updateStatus) + require.Nil(t, err) + + _, resp = th.SystemAdminClient.DownloadJob(job.Id) + CheckNotFoundStatus(t, resp) + + // Now we stub the results of the job into the same directory and try to download it again + // This time we should successfully retrieve the results without any error + filePath = "./data/export/" + job.Id + "/csv_export.zip" + mkdirAllErr = os.MkdirAll(filepath.Dir(filePath), 0770) + require.Nil(t, mkdirAllErr) + os.Create(filePath) + + _, resp = th.SystemAdminClient.DownloadJob(job.Id) + require.Nil(t, resp.Error) +} + func TestCancelJob(t *testing.T) { th := Setup(t) defer th.TearDown() diff --git a/i18n/en.json b/i18n/en.json index 33be9fa30a..bb68d48256 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1452,6 +1452,10 @@ "id": "api.io_error", "translation": "input/output error" }, + { + "id": "api.job.unable_to_download_job", + "translation": "Unable to download this job" + }, { "id": "api.ldap_group.not_found", "translation": "ldap group not found" @@ -3954,6 +3958,10 @@ "id": "app.import.validate_user_teams_import_data.team_name_missing.error", "translation": "Team name missing from User's Team Membership." }, + { + "id": "app.job.download_export_results_not_enabled", + "translation": "DownloadExportResults in config.json is false. Please set this to true to download the results of this job." + }, { "id": "app.notification.body.intro.direct.full", "translation": "You have a new Direct Message." diff --git a/model/client4.go b/model/client4.go index ae5398cf52..2557460b2f 100644 --- a/model/client4.go +++ b/model/client4.go @@ -4698,6 +4698,21 @@ func (c *Client4) CancelJob(jobId string) (bool, *Response) { return CheckStatusOK(r), BuildResponse(r) } +// DownloadJob downloads the results of the job +func (c *Client4) DownloadJob(jobId string) ([]byte, *Response) { + r, appErr := c.DoApiGet(c.GetJobsRoute()+fmt.Sprintf("/%v/download", jobId), "") + if appErr != nil { + return nil, BuildErrorResponse(r, appErr) + } + defer closeBody(r) + + data, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, BuildErrorResponse(r, NewAppError("GetFile", "model.client.read_job_result_file.app_error", nil, err.Error(), r.StatusCode)) + } + return data, BuildResponse(r) +} + // Roles Section // GetRole gets a single role by ID. diff --git a/model/config.go b/model/config.go index 6c5776a9dc..6634107583 100644 --- a/model/config.go +++ b/model/config.go @@ -2615,11 +2615,12 @@ func (s *GlobalRelayMessageExportSettings) SetDefaults() { } type MessageExportSettings struct { - EnableExport *bool - ExportFormat *string - DailyRunTime *string - ExportFromTimestamp *int64 - BatchSize *int + EnableExport *bool + DownloadExportResults *bool + ExportFormat *string + DailyRunTime *string + ExportFromTimestamp *int64 + BatchSize *int // formatter-specific settings - these are only expected to be non-nil if ExportFormat is set to the associated format GlobalRelaySettings *GlobalRelayMessageExportSettings @@ -2630,6 +2631,10 @@ func (s *MessageExportSettings) SetDefaults() { s.EnableExport = NewBool(false) } + if s.DownloadExportResults == nil { + s.DownloadExportResults = NewBool(false) + } + if s.ExportFormat == nil { s.ExportFormat = NewString(COMPLIANCE_EXPORT_TYPE_ACTIANCE) }