[MM-56616] Changes for the DM for batch reporting (#26019)

* [MM-56616] Changes for the DM for batch reporting

* Use requesting user's locale

* Fix lint

* Remove unnecessary test

* Move back to file attachment

* Add default API case

* Fix i18n

* Hardcode the CSV string

api/v4/source/reports.yaml

@@ -195,42 +195,3 @@
           $ref: "#/components/responses/Forbidden"
         "500":
           $ref: "#/components/responses/InternalServerError"
-  /api/v4/reports/export/{report_id}:
-    get:
-      tags:
-        - reports
-      summary: Retrieves a compiled report file for the given report_id
-      description: >
-        Retrieves a compiled report file for the given report_id.
-
-        Must be a system admin to invoke this API.
-        
-        ##### Permissions
-        
-        Requires `sysconsole_read_user_management_users`.
-      
-      operationId: RetrieveBatchReportFile
-      parameters:
-        - name: report_id
-          in: path
-          description: Report ID for the given batch job
-          required: true
-          schema:
-            type: string
-        - name: format
-          in: query
-          description: The format of the report generated (one of "csv")
-          required: true
-          schema:
-            type: string
-      responses:
-        "400":
-          $ref: "#/components/responses/BadRequest"
-        "401":
-          $ref: "#/components/responses/Unauthorized"
-        "403":
-          $ref: "#/components/responses/Forbidden"
-        "404":
-          $ref: "#/components/responses/NotFound"
-        "500":
-          $ref: "#/components/responses/InternalServerError"

server/channels/api4/report.go

@@ -18,8 +18,6 @@ func (api *API) InitReports() {
 	api.BaseRoutes.Reports.Handle("/users", api.APISessionRequired(getUsersForReporting)).Methods("GET")
 	api.BaseRoutes.Reports.Handle("/users/count", api.APISessionRequired(getUserCountForReporting)).Methods("GET")
 	api.BaseRoutes.Reports.Handle("/users/export", api.APISessionRequired(startUsersBatchExport)).Methods("POST")
-
-	api.BaseRoutes.Reports.Handle("/export/{report_id:[A-Za-z0-9]+}", api.APISessionRequired(retrieveBatchReportFile)).Methods("GET")
 }
 
 func getUsersForReporting(c *Context, w http.ResponseWriter, r *http.Request) {
@@ -82,8 +80,13 @@ func startUsersBatchExport(c *Context, w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	startAt, endAt := model.GetReportDateRange(r.URL.Query().Get("date_range"), time.Now())
-	if err := c.App.StartUsersBatchExport(c.AppContext, startAt, endAt); err != nil {
+	dateRange := r.URL.Query().Get("date_range")
+	if dateRange == "" {
+		dateRange = "all_time"
+	}
+
+	startAt, endAt := model.GetReportDateRange(dateRange, time.Now())
+	if err := c.App.StartUsersBatchExport(c.AppContext, dateRange, startAt, endAt); err != nil {
 		c.Err = err
 		return
 	}
@@ -91,35 +94,6 @@ func startUsersBatchExport(c *Context, w http.ResponseWriter, r *http.Request) {
 	ReturnStatusOK(w)
 }
 
-func retrieveBatchReportFile(c *Context, w http.ResponseWriter, r *http.Request) {
-	if !(c.IsSystemAdmin()) {
-		c.SetPermissionError(model.PermissionSysconsoleReadUserManagementUsers)
-		return
-	}
-
-	reportId := c.Params.ReportId
-	if !model.IsValidId(reportId) {
-		c.Err = model.NewAppError("retrieveBatchReportFile", "api.retrieveBatchReportFile.invalid_report_id", nil, "", http.StatusBadRequest)
-		return
-	}
-
-	format := r.URL.Query().Get("format")
-	if !model.IsValidReportExportFormat(format) {
-		c.Err = model.NewAppError("retrieveBatchReportFile", "api.retrieveBatchReportFile.invalid_format", nil, "", http.StatusBadRequest)
-		return
-	}
-
-	file, name, err := c.App.RetrieveBatchReport(reportId, format)
-	if err != nil {
-		c.Err = err
-		return
-	}
-	defer file.Close()
-
-	w.Header().Set("Content-Type", "text/csv")
-	http.ServeContent(w, r, name, time.Time{}, file)
-}
-
 func fillReportingBaseOptions(values url.Values) model.ReportingBaseOptions {
 	sortColumn := "Username"
 	if values.Get("sort_column") != "" {

server/channels/app/app_iface.go

@@ -1025,7 +1025,6 @@ type AppIface interface {
 	RestoreTeam(teamID string) *model.AppError
 	RestrictUsersGetByPermissions(c request.CTX, userID string, options *model.UserGetOptions) (*model.UserGetOptions, *model.AppError)
 	RestrictUsersSearchByPermissions(c request.CTX, userID string, options *model.UserSearchOptions) (*model.UserSearchOptions, *model.AppError)
-	RetrieveBatchReport(reportID string, format string) (filestore.ReadCloseSeeker, string, *model.AppError)
 	ReturnSessionToPool(session *model.Session)
 	RevokeAccessToken(c request.CTX, token string) *model.AppError
 	RevokeAllSessions(c request.CTX, userID string) *model.AppError
@@ -1082,7 +1081,7 @@ type AppIface interface {
 	SendPasswordReset(email string, siteURL string) (bool, *model.AppError)
 	SendPaymentFailedEmail(failedPayment *model.FailedPayment) *model.AppError
 	SendPersistentNotifications() error
-	SendReportToUser(rctx request.CTX, userID string, jobId string, format string) *model.AppError
+	SendReportToUser(rctx request.CTX, job *model.Job, format string) *model.AppError
 	SendTestPushNotification(deviceID string) string
 	SendUpgradeConfirmationEmail(isYearly bool) *model.AppError
 	ServeInterPluginRequest(w http.ResponseWriter, r *http.Request, sourcePluginId, destinationPluginId string)
@@ -1127,7 +1126,7 @@ type AppIface interface {
 	SlackImport(c request.CTX, fileData multipart.File, fileSize int64, teamID string) (*model.AppError, *bytes.Buffer)
 	SoftDeleteTeam(teamID string) *model.AppError
 	Srv() *Server
-	StartUsersBatchExport(rctx request.CTX, startAt int64, endAt int64) *model.AppError
+	StartUsersBatchExport(rctx request.CTX, dateRange string, startAt int64, endAt int64) *model.AppError
 	SubmitInteractiveDialog(c request.CTX, request model.SubmitDialogRequest) (*model.SubmitDialogResponse, *model.AppError)
 	SwitchEmailToLdap(c request.CTX, email, password, code, ldapLoginId, ldapPassword string) (string, *model.AppError)
 	SwitchEmailToOAuth(c request.CTX, w http.ResponseWriter, r *http.Request, email, password, code, service string) (string, *model.AppError)

server/channels/app/opentracing/opentracing_layer.go

@@ -14662,28 +14662,6 @@ func (a *OpenTracingAppLayer) RestrictUsersSearchByPermissions(c request.CTX, us
 	return resultVar0, resultVar1
 }
 
-func (a *OpenTracingAppLayer) RetrieveBatchReport(reportID string, format string) (filestore.ReadCloseSeeker, string, *model.AppError) {
-	origCtx := a.ctx
-	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.RetrieveBatchReport")
-
-	a.ctx = newCtx
-	a.app.Srv().Store().SetContext(newCtx)
-	defer func() {
-		a.app.Srv().Store().SetContext(origCtx)
-		a.ctx = origCtx
-	}()
-
-	defer span.Finish()
-	resultVar0, resultVar1, resultVar2 := a.app.RetrieveBatchReport(reportID, format)
-
-	if resultVar2 != nil {
-		span.LogFields(spanlog.Error(resultVar2))
-		ext.Error.Set(span, true)
-	}
-
-	return resultVar0, resultVar1, resultVar2
-}
-
 func (a *OpenTracingAppLayer) ReturnSessionToPool(session *model.Session) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.ReturnSessionToPool")
@@ -15960,7 +15938,7 @@ func (a *OpenTracingAppLayer) SendPersistentNotifications() error {
 	return resultVar0
 }
 
-func (a *OpenTracingAppLayer) SendReportToUser(rctx request.CTX, userID string, jobId string, format string) *model.AppError {
+func (a *OpenTracingAppLayer) SendReportToUser(rctx request.CTX, job *model.Job, format string) *model.AppError {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.SendReportToUser")
 
@@ -15972,7 +15950,7 @@ func (a *OpenTracingAppLayer) SendReportToUser(rctx request.CTX, userID string,
 	}()
 
 	defer span.Finish()
-	resultVar0 := a.app.SendReportToUser(rctx, userID, jobId, format)
+	resultVar0 := a.app.SendReportToUser(rctx, job, format)
 
 	if resultVar0 != nil {
 		span.LogFields(spanlog.Error(resultVar0))
@@ -16923,7 +16901,7 @@ func (a *OpenTracingAppLayer) SoftDeleteTeam(teamID string) *model.AppError {
 	return resultVar0
 }
 
-func (a *OpenTracingAppLayer) StartUsersBatchExport(rctx request.CTX, startAt int64, endAt int64) *model.AppError {
+func (a *OpenTracingAppLayer) StartUsersBatchExport(rctx request.CTX, dateRange string, startAt int64, endAt int64) *model.AppError {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.StartUsersBatchExport")
 
@@ -16935,7 +16913,7 @@ func (a *OpenTracingAppLayer) StartUsersBatchExport(rctx request.CTX, startAt in
 	}()
 
 	defer span.Finish()
-	resultVar0 := a.app.StartUsersBatchExport(rctx, startAt, endAt)
+	resultVar0 := a.app.StartUsersBatchExport(rctx, dateRange, startAt, endAt)
 
 	if resultVar0 != nil {
 		span.LogFields(spanlog.Error(resultVar0))

server/channels/app/report.go

@@ -14,7 +14,6 @@ import (
 	"github.com/mattermost/mattermost/server/public/shared/i18n"
 	"github.com/mattermost/mattermost/server/public/shared/mlog"
 	"github.com/mattermost/mattermost/server/public/shared/request"
-	"github.com/mattermost/mattermost/server/v8/platform/shared/filestore"
 )
 
 func (a *App) SaveReportChunk(format string, prefix string, count int, reportData []model.ReportableObject) *model.AppError {
@@ -84,26 +83,55 @@ func (a *App) compileCSVChunks(prefix string, numberOfChunks int, headers []stri
 	return nil
 }
 
-func (a *App) SendReportToUser(rctx request.CTX, userID string, jobId string, format string) *model.AppError {
+func (a *App) SendReportToUser(rctx request.CTX, job *model.Job, format string) *model.AppError {
+	requestingUserId := job.Data["requesting_user_id"]
+	if requestingUserId == "" {
+		return model.NewAppError("SendReportToUser", "app.report.send_report_to_user.missing_user_id", nil, "", http.StatusInternalServerError)
+	}
+	dateRange := job.Data["date_range"]
+	if dateRange == "" {
+		return model.NewAppError("SendReportToUser", "app.report.send_report_to_user.missing_date_range", nil, "", http.StatusInternalServerError)
+	}
+
 	systemBot, err := a.GetSystemBot()
 	if err != nil {
 		return err
 	}
 
-	channel, err := a.GetOrCreateDirectChannel(request.EmptyContext(a.Log()), userID, systemBot.UserId)
+	path := makeCompiledFilePath(job.Id, format)
+	size, err := a.FileSize(path)
+	if err != nil {
+		return err
+	}
+	fileInfo, fileErr := a.Srv().Store().FileInfo().Save(rctx, &model.FileInfo{
+		Name:      makeCompiledFilename(job.Id, format),
+		Extension: format,
+		Size:      size,
+		Path:      path,
+		CreatorId: systemBot.UserId,
+	})
+	if fileErr != nil {
+		return model.NewAppError("SendReportToUser", "app.report.send_report_to_user.failed_to_save", nil, "", http.StatusInternalServerError).Wrap(fileErr)
+	}
+
+	channel, err := a.GetOrCreateDirectChannel(request.EmptyContext(a.Log()), requestingUserId, systemBot.UserId)
 	if err != nil {
 		return err
 	}
 
+	user, err := a.GetUser(requestingUserId)
+	if err != nil {
+		return err
+	}
+	T := i18n.GetUserTranslations(user.Locale)
 	post := &model.Post{
 		ChannelId: channel.Id,
-		Message:   i18n.T("app.report.send_report_to_user.export_finished", map[string]string{"Link": a.GetSiteURL() + "/api/v4/reports/export/" + jobId + "?format=" + format}),
-		Type:      model.PostTypeAdminReport,
-		UserId:    systemBot.UserId,
-		Props: model.StringInterface{
-			"reportId": jobId,
-			"format":   format,
-		},
+		Message: T("app.report.send_report_to_user.export_finished", map[string]string{
+			"DateRange": getTranslatedDateRange(dateRange),
+		}),
+		Type:    model.PostTypeDefault,
+		UserId:  systemBot.UserId,
+		FileIds: []string{fileInfo.Id},
 	}
 
 	_, err = a.CreatePost(rctx, post, channel, false, true)
@@ -168,13 +196,14 @@ func (a *App) GetUserCountForReport(filter *model.UserReportOptions) (*int64, *m
 	return &count, nil
 }
 
-func (a *App) StartUsersBatchExport(rctx request.CTX, startAt int64, endAt int64) *model.AppError {
+func (a *App) StartUsersBatchExport(rctx request.CTX, dateRange string, startAt int64, endAt int64) *model.AppError {
 	if license := a.Srv().License(); license == nil || (license.SkuShortName != model.LicenseShortSkuProfessional && license.SkuShortName != model.LicenseShortSkuEnterprise) {
 		return model.NewAppError("StartUsersBatchExport", "app.report.start_users_batch_export.license_error", nil, "", http.StatusBadRequest)
 	}
 
 	options := map[string]string{
 		"requesting_user_id": rctx.Session().UserId,
+		"date_range":         dateRange,
 		"start_at":           strconv.FormatInt(startAt, 10),
 		"end_at":             strconv.FormatInt(endAt, 10),
 	}
@@ -186,7 +215,7 @@ func (a *App) StartUsersBatchExport(rctx request.CTX, startAt int64, endAt int64
 		return err
 	}
 	for _, job := range pendingJobs {
-		if job.Data["start_at"] == options["start_at"] && job.Data["end_at"] == options["end_at"] && job.Data["requesting_user_id"] == rctx.Session().UserId {
+		if job.Data["date_range"] == options["date_range"] && job.Data["requesting_user_id"] == rctx.Session().UserId {
 			return model.NewAppError("StartUsersBatchExport", "app.report.start_users_batch_export.job_exists", nil, "", http.StatusBadRequest)
 		}
 	}
@@ -196,7 +225,7 @@ func (a *App) StartUsersBatchExport(rctx request.CTX, startAt int64, endAt int64
 		return err
 	}
 	for _, job := range inProgressJobs {
-		if job.Data["start_at"] == options["start_at"] && job.Data["end_at"] == options["end_at"] && job.Data["requesting_user_id"] == rctx.Session().UserId {
+		if job.Data["date_range"] == options["date_range"] && job.Data["requesting_user_id"] == rctx.Session().UserId {
 			return model.NewAppError("StartUsersBatchExport", "app.report.start_users_batch_export.job_exists", nil, "", http.StatusBadRequest)
 		}
 	}
@@ -219,9 +248,15 @@ func (a *App) StartUsersBatchExport(rctx request.CTX, startAt int64, endAt int64
 			return
 		}
 
+		user, err := a.GetUser(rctx.Session().UserId)
+		if err != nil {
+			rctx.Logger().Error("Failed to get the user", mlog.Err(err))
+			return
+		}
+		T := i18n.GetUserTranslations(user.Locale)
 		post := &model.Post{
 			ChannelId: channel.Id,
-			Message:   i18n.T("app.report.start_users_batch_export.started_export"),
+			Message:   T("app.report.start_users_batch_export.started_export", map[string]string{"DateRange": getTranslatedDateRange(dateRange)}),
 			Type:      model.PostTypeDefault,
 			UserId:    systemBot.UserId,
 		}
@@ -234,16 +269,15 @@ func (a *App) StartUsersBatchExport(rctx request.CTX, startAt int64, endAt int64
 	return nil
 }
 
-func (a *App) RetrieveBatchReport(reportID string, format string) (filestore.ReadCloseSeeker, string, *model.AppError) {
-	if license := a.Srv().License(); license == nil || (license.SkuShortName != model.LicenseShortSkuProfessional && license.SkuShortName != model.LicenseShortSkuEnterprise) {
-		return nil, "", model.NewAppError("RetrieveBatchReport", "app.report.retrieve_batch_report.license_error", nil, "", http.StatusBadRequest)
+func getTranslatedDateRange(dateRange string) string {
+	switch dateRange {
+	case model.ReportDurationLast30Days:
+		return i18n.T("app.report.date_range.last_30_days")
+	case model.ReportDurationPreviousMonth:
+		return i18n.T("app.report.date_range.previous_month")
+	case model.ReportDurationLast6Months:
+		return i18n.T("app.report.date_range.last_6_months")
+	default:
+		return i18n.T("app.report.date_range.all_time")
 	}
-
-	filePath := makeCompiledFilePath(reportID, format)
-	reader, err := a.FileReader(filePath)
-	if err != nil {
-		return nil, "", err
-	}
-
-	return reader, makeCompiledFilename(reportID, format), nil
 }

server/channels/jobs/batch_report_worker.go

@@ -18,7 +18,7 @@ import (
 type BatchReportWorkerAppIFace interface {
 	SaveReportChunk(format string, prefix string, count int, reportData []model.ReportableObject) *model.AppError
 	CompileReportChunks(format string, prefix string, numberOfChunks int, headers []string) *model.AppError
-	SendReportToUser(rctx request.CTX, userID string, jobId string, format string) *model.AppError
+	SendReportToUser(rctx request.CTX, job *model.Job, format string) *model.AppError
 	CleanupReportChunks(format string, prefix string, numberOfChunks int) *model.AppError
 }
 
@@ -113,10 +113,6 @@ func (worker *BatchReportWorker) processChunk(job *model.Job, reportData []model
 }
 
 func (worker *BatchReportWorker) complete(rctx request.CTX, job *model.Job) error {
-	requestingUserId := job.Data["requesting_user_id"]
-	if requestingUserId == "" {
-		return errors.New("No user to send the report to")
-	}
 	fileCount, err := getFileCount(job.Data)
 	if err != nil {
 		return err
@@ -131,7 +127,7 @@ func (worker *BatchReportWorker) complete(rctx request.CTX, job *model.Job) erro
 		worker.app.CleanupReportChunks(worker.reportFormat, job.Id, fileCount)
 	}()
 
-	if appErr = worker.app.SendReportToUser(rctx, requestingUserId, job.Id, worker.reportFormat); appErr != nil {
+	if appErr = worker.app.SendReportToUser(rctx, job, worker.reportFormat); appErr != nil {
 		return appErr
 	}
 

server/channels/jobs/batch_report_worker_test.go

@@ -23,7 +23,7 @@ func (rma *ReportMockApp) SaveReportChunk(format string, prefix string, count in
 func (rma *ReportMockApp) CompileReportChunks(format string, prefix string, numberOfChunks int, headers []string) *model.AppError {
 	return nil
 }
-func (rma *ReportMockApp) SendReportToUser(rctx request.CTX, userID string, jobId string, format string) *model.AppError {
+func (rma *ReportMockApp) SendReportToUser(rctx request.CTX, job *model.Job, format string) *model.AppError {
 	return nil
 }
 func (rma *ReportMockApp) CleanupReportChunks(format string, prefix string, numberOfChunks int) *model.AppError {
@@ -129,21 +129,4 @@ func TestBatchReportWorker(t *testing.T) {
 
 		th.WaitForJobStatus(t, job, model.JobStatusError)
 	})
-
-	t.Run("should fail if there is no user id to send the report to", func(t *testing.T) {
-		th := Setup(t).InitBasic()
-		defer th.TearDown()
-
-		var worker model.Worker
-		var job *model.Job
-		worker, job = setupBatchWorker(t, th, func(data model.StringMap) ([]model.ReportableObject, model.StringMap, bool, error) {
-			go worker.Stop() // Shut down the worker right after this
-			return []model.ReportableObject{}, make(model.StringMap), true, nil
-		})
-
-		// Queue the work to be done
-		worker.JobChannel() <- *job
-
-		th.WaitForJobStatus(t, job, model.JobStatusError)
-	})
 }

server/i18n/en.json

@@ -2706,14 +2706,6 @@
     "id": "api.restricted_system_admin",
     "translation": "This action is forbidden to a restricted system admin."
   },
-  {
-    "id": "api.retrieveBatchReportFile.invalid_format",
-    "translation": "Report format is invalid."
-  },
-  {
-    "id": "api.retrieveBatchReportFile.invalid_report_id",
-    "translation": "Report ID is invalid."
-  },
   {
     "id": "api.roles.get_multiple_by_name_too_many.request_error",
     "translation": "Unable to get that many roles by name. Only {{.MaxNames}} roles can be requested at once."
@@ -6682,6 +6674,22 @@
     "id": "app.recover.save.app_error",
     "translation": "Unable to save the token."
   },
+  {
+    "id": "app.report.date_range.all_time",
+    "translation": "all time"
+  },
+  {
+    "id": "app.report.date_range.last_30_days",
+    "translation": "the last 30 days"
+  },
+  {
+    "id": "app.report.date_range.last_6_months",
+    "translation": "the last 6 months"
+  },
+  {
+    "id": "app.report.date_range.previous_month",
+    "translation": "the previous month"
+  },
   {
     "id": "app.report.get_user_count_for_report.store_error",
     "translation": "Failed to fetch user count."
@@ -6691,12 +6699,20 @@
     "translation": "Failed to fetch user report."
   },
   {
-    "id": "app.report.retrieve_batch_report.license_error",
-    "translation": "Batch reporting export only available to Pro and Enterprise."
+    "id": "app.report.send_report_to_user.export_finished",
+    "translation": "Your export is ready. The CSV file contains user data for {{.DateRange}}. Click on the link below to download the report."
   },
   {
-    "id": "app.report.send_report_to_user.export_finished",
-    "translation": "Report processing is finished. You can download the report [here]({{.Link}})"
+    "id": "app.report.send_report_to_user.failed_to_save",
+    "translation": "Failed to save file info."
+  },
+  {
+    "id": "app.report.send_report_to_user.missing_date_range",
+    "translation": "Missing date range"
+  },
+  {
+    "id": "app.report.send_report_to_user.missing_user_id",
+    "translation": "No user id to send the report to"
   },
   {
     "id": "app.report.start_users_batch_export.job_exists",
@@ -6708,7 +6724,7 @@
   },
   {
     "id": "app.report.start_users_batch_export.started_export",
-    "translation": "You have requested the export of user data. A CSV will be delivered to you when the export is complete."
+    "translation": "You've started an export of user data for {{.DateRange}}. When the export is complete, a CSV file will be delivered to you in this direct message."
   },
   {
     "id": "app.role.check_roles_exist.role_not_found",

server/public/model/post.go

@@ -52,7 +52,6 @@ const (
 	PostTypeMe                     = "me"
 	PostCustomTypePrefix           = "custom_"
 	PostTypeReminder               = "reminder"
-	PostTypeAdminReport            = "system_admin_report"
 
 	PostFileidsMaxRunes   = 300
 	PostFilenamesMaxRunes = 4000
@@ -451,8 +450,7 @@ func (o *Post) IsValid(maxPostSize int) *AppError {
 		PostTypeReminder,
 		PostTypeMe,
 		PostTypeWrangler,
-		PostTypeGMConvertedToChannel,
-		PostTypeAdminReport:
+		PostTypeGMConvertedToChannel:
 	default:
 		if !strings.HasPrefix(o.Type, PostCustomTypePrefix) {
 			return NewAppError("Post.IsValid", "model.post.is_valid.type.app_error", nil, "id="+o.Type, http.StatusBadRequest)

server/public/model/report.go

@@ -12,6 +12,7 @@ import (
 )
 
 const (
+	ReportDurationAllTime       = "all_time"
 	ReportDurationLast30Days    = "last_30_days"
 	ReportDurationPreviousMonth = "previous_month"
 	ReportDurationLast6Months   = "last_6_months"