[MM-54770] Add ability to export profile pictures and fix importing them (#25042)

* add ability to export pp and fix import

* remove unused nopSeeker

* remove debug log

* fix shadow vars

* generate a warning instead of an error when unable to export profile picture

* fix merge conflicts

---------

Co-authored-by: Mattermost Build <build@mattermost.com>

server/channels/app/app_iface.go

@@ -220,6 +220,8 @@ type AppIface interface {
 	GetPostsUsage() (int64, *model.AppError)
 	// GetProductNotices is called from the frontend to fetch the product notices that are relevant to the caller
 	GetProductNotices(c request.CTX, userID, teamID string, client model.NoticeClientType, clientVersion string, locale string) (model.NoticeMessages, *model.AppError)
+	// GetProfileImagePaths returns the paths to the profile images for the given user IDs if such a profile image exists.
+	GetProfileImagePath(user *model.User) (string, *model.AppError)
 	// GetPublicKey will return the actual public key saved in the `name` file.
 	GetPublicKey(name string) ([]byte, *model.AppError)
 	// GetSanitizedConfig gets the configuration for a system admin without any secrets.

server/channels/app/export.go

@@ -102,7 +102,8 @@ func (a *App) BulkExport(ctx request.CTX, writer io.Writer, outPath string, job
 	}
 
 	ctx.Logger().Info("Bulk export: exporting users")
-	if err = a.exportAllUsers(ctx, job, writer, opts.IncludeArchivedChannels); err != nil {
+	profilePictures, err := a.exportAllUsers(ctx, job, writer, opts.IncludeArchivedChannels, opts.IncludeProfilePictures)
+	if err != nil {
 		return err
 	}
 
@@ -150,6 +151,15 @@ func (a *App) BulkExport(ctx request.CTX, writer io.Writer, outPath string, job
 		updateJobProgress(ctx.Logger(), a.Srv().Store(), job, "attachments_exported", len(attachments)+len(directAttachments)+len(emojiPaths))
 	}
 
+	if opts.IncludeProfilePictures {
+		for _, profilePicture := range profilePictures {
+			if err := a.exportFile(outPath, profilePicture, zipWr); err != nil {
+				ctx.Logger().Warn("Unable to export profile picture", mlog.String("profile_picture", profilePicture), mlog.Err(err))
+			}
+		}
+		updateJobProgress(ctx.Logger(), a.Srv().Store(), job, "profile_pictures_exported", len(profilePictures))
+	}
+
 	return nil
 }
 
@@ -257,14 +267,15 @@ func (a *App) exportAllChannels(ctx request.CTX, job *model.Job, writer io.Write
 	return nil
 }
 
-func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer, includeArchivedChannels bool) *model.AppError {
+func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer, includeArchivedChannels, includeProfilePictures bool) ([]string, *model.AppError) {
 	afterId := strings.Repeat("0", 26)
 	cnt := 0
+	profilePictures := []string{}
 	for {
 		users, err := a.Srv().Store().User().GetAllAfter(1000, afterId)
 
 		if err != nil {
-			return model.NewAppError("exportAllUsers", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
+			return profilePictures, model.NewAppError("exportAllUsers", "app.user.get.app_error", nil, "", http.StatusInternalServerError).Wrap(err)
 		}
 
 		if len(users) == 0 {
@@ -280,7 +291,7 @@ func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer,
 			exportedPrefs := make(map[string]*string)
 			allPrefs, err := a.GetPreferencesForUser(ctx, user.Id)
 			if err != nil {
-				return err
+				return profilePictures, err
 			}
 			for _, pref := range allPrefs {
 				// We need to manage the special cases
@@ -316,23 +327,35 @@ func (a *App) exportAllUsers(ctx request.CTX, job *model.Job, writer io.Writer,
 
 			userLine := ImportLineFromUser(user, exportedPrefs)
 
+			if includeProfilePictures {
+				var pp string
+				pp, err = a.GetProfileImagePath(user)
+				if err != nil {
+					return profilePictures, err
+				}
+				if pp != "" {
+					userLine.User.ProfileImage = &pp
+					profilePictures = append(profilePictures, pp)
+				}
+			}
+
 			userLine.User.NotifyProps = a.buildUserNotifyProps(user.NotifyProps)
 
 			// Do the Team Memberships.
 			members, err := a.buildUserTeamAndChannelMemberships(ctx, user.Id, includeArchivedChannels)
 			if err != nil {
-				return err
+				return profilePictures, err
 			}
 
 			userLine.User.Teams = members
 
 			if err := a.exportWriteLine(writer, userLine); err != nil {
-				return err
+				return profilePictures, err
 			}
 		}
 	}
 
-	return nil
+	return profilePictures, nil
 }
 
 func (a *App) buildUserTeamAndChannelMemberships(c request.CTX, userID string, includeArchivedChannels bool) (*[]imports.UserTeamImportData, *model.AppError) {

server/channels/app/imaging/decode.go

@@ -113,7 +113,7 @@ func (d *Decoder) DecodeConfig(rd io.Reader) (image.Config, string, error) {
 // GetDimensions returns the dimensions for the given encoded image data.
 func GetDimensions(imageData io.Reader) (int, int, error) {
 	cfg, _, err := image.DecodeConfig(imageData)
-	if seeker, ok := imageData.(io.ReadSeeker); ok {
+	if seeker, ok := imageData.(io.Seeker); ok {
 		defer seeker.Seek(0, 0)
 	}
 	return cfg.Width, cfg.Height, err

server/channels/app/import_functions.go

@@ -619,18 +619,34 @@ func (a *App) importUser(rctx request.CTX, data *imports.UserImportData, dryRun
 	}
 
 	if data.ProfileImage != nil {
-		var file io.ReadCloser
+		var file io.ReadSeeker
 		var err error
 		if data.ProfileImageData != nil {
-			file, err = data.ProfileImageData.Open()
+			// *zip.File does not support Seek, and we need a seeker to reset the cursor position after checking the picture dimension
+			var f io.ReadCloser
+			f, err = data.ProfileImageData.Open()
+			if err != nil {
+				rctx.Logger().Warn("Unable to open the profile image data.", mlog.Err(err))
+			} else {
+				limitedReader := io.LimitReader(f, *a.Config().FileSettings.MaxFileSize)
+				var b []byte
+				b, err = io.ReadAll(limitedReader)
+				if err != nil {
+					rctx.Logger().Warn("Unable to read all bytes from profile picture.", mlog.Err(err))
+				} else {
+					file = bytes.NewReader(b)
+				}
+			}
 		} else {
 			file, err = os.Open(*data.ProfileImage)
+			if err != nil {
+				rctx.Logger().Warn("Unable to open the profile image.", mlog.Err(err))
+			} else {
+				defer file.(*os.File).Close()
+			}
 		}
 
-		if err != nil {
-			rctx.Logger().Warn("Unable to open the profile image.", mlog.Err(err))
-		} else {
-			defer file.Close()
+		if file != nil {
 			if limitErr := checkImageLimits(file, *a.Config().FileSettings.MaxImageResolution); limitErr != nil {
 				return model.NewAppError("SetProfileImage", "api.user.upload_profile_user.check_image_limits.app_error", nil, "", http.StatusBadRequest)
 			}

server/channels/app/imports/import_validators.go

@@ -190,9 +190,9 @@ func ValidateChannelImportData(data *ChannelImportData) *model.AppError {
 }
 
 func ValidateUserImportData(data *UserImportData) *model.AppError {
-	if data.ProfileImage != nil {
+	if data.ProfileImage != nil && data.ProfileImageData == nil {
 		if _, err := os.Stat(*data.ProfileImage); os.IsNotExist(err) {
-			return model.NewAppError("BulkImport", "app.import.validate_user_import_data.profile_image.error", nil, "", http.StatusBadRequest)
+			return model.NewAppError("BulkImport", "app.import.validate_user_import_data.profile_image.error", nil, "", http.StatusBadRequest).Wrap(err)
 		}
 	}
 

server/channels/app/opentracing/opentracing_layer.go

@@ -8603,6 +8603,28 @@ func (a *OpenTracingAppLayer) GetProfileImage(user *model.User) ([]byte, bool, *
 	return resultVar0, resultVar1, resultVar2
 }
 
+func (a *OpenTracingAppLayer) GetProfileImagePath(user *model.User) (string, *model.AppError) {
+	origCtx := a.ctx
+	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetProfileImagePath")
+
+	a.ctx = newCtx
+	a.app.Srv().Store().SetContext(newCtx)
+	defer func() {
+		a.app.Srv().Store().SetContext(origCtx)
+		a.ctx = origCtx
+	}()
+
+	defer span.Finish()
+	resultVar0, resultVar1 := a.app.GetProfileImagePath(user)
+
+	if resultVar1 != nil {
+		span.LogFields(spanlog.Error(resultVar1))
+		ext.Error.Set(span, true)
+	}
+
+	return resultVar0, resultVar1
+}
+
 func (a *OpenTracingAppLayer) GetPublicChannelsByIdsForTeam(c request.CTX, teamID string, channelIDs []string) (model.ChannelList, *model.AppError) {
 	origCtx := a.ctx
 	span, newCtx := tracing.StartSpanWithParentByContext(a.ctx, "app.GetPublicChannelsByIdsForTeam")

server/channels/app/server.go

@@ -1748,7 +1748,7 @@ func (s *Server) GetProfileImage(user *model.User) ([]byte, bool, *model.AppErro
 		return img, false, nil
 	}
 
-	path := "users/" + user.Id + "/profile.png"
+	path := getProfileImagePath(user.Id)
 
 	data, err := s.ReadFile(path)
 	if err != nil {

server/channels/app/user.go

@@ -789,6 +789,25 @@ func (a *App) DeactivateMfa(userID string) *model.AppError {
 	return nil
 }
 
+// GetProfileImagePaths returns the paths to the profile images for the given user IDs if such a profile image exists.
+func (a *App) GetProfileImagePath(user *model.User) (string, *model.AppError) {
+	path := getProfileImagePath(user.Id)
+	exist, err := a.ch.srv.FileBackend().FileExists(path)
+	if err != nil {
+		return "", model.NewAppError(
+			"GetProfileImagePath",
+			"api.user.get_profile_image_path.app_error",
+			nil,
+			"",
+			http.StatusInternalServerError,
+		).Wrap(err)
+	}
+	if !exist {
+		return "", nil
+	}
+	return path, nil
+}
+
 func (a *App) GetProfileImage(user *model.User) ([]byte, bool, *model.AppError) {
 	return a.ch.srv.GetProfileImage(user)
 }

server/channels/jobs/export_process/worker.go

@@ -43,6 +43,11 @@ func MakeWorker(jobServer *jobs.JobServer, app AppIface) *jobs.SimpleWorker {
 			opts.IncludeArchivedChannels = true
 		}
 
+		includeProfilePictures, ok := job.Data["include_profile_pictures"]
+		if ok && includeProfilePictures == "true" {
+			opts.IncludeProfilePictures = true
+		}
+
 		outPath := *app.Config().ExportSettings.Directory
 		exportFilename := job.Id + "_export.zip"
 

server/cmd/mattermost/commands/export.go

@@ -82,6 +82,7 @@ func init() {
 
 	BulkExportCmd.Flags().Bool("all-teams", true, "Export all teams from the server.")
 	BulkExportCmd.Flags().Bool("with-archived-channels", false, "Also exports archived channels.")
+	BulkExportCmd.Flags().Bool("with-profile-pictures", false, "Also exports profile pictures.")
 	BulkExportCmd.Flags().Bool("attachments", false, "Also export file attachments.")
 	BulkExportCmd.Flags().Bool("archive", false, "Outputs a single archive file.")
 
@@ -240,6 +241,11 @@ func bulkExportCmdF(command *cobra.Command, args []string) error {
 		return errors.Wrap(err, "with-archived-channels flag error")
 	}
 
+	includeProfilePictures, err := command.Flags().GetBool("with-profile-pictures")
+	if err != nil {
+		return errors.Wrap(err, "with-profile-pictures flag error")
+	}
+
 	fileWriter, err := os.Create(args[0])
 	if err != nil {
 		return err
@@ -255,6 +261,7 @@ func bulkExportCmdF(command *cobra.Command, args []string) error {
 	opts.IncludeAttachments = attachments
 	opts.CreateArchive = archive
 	opts.IncludeArchivedChannels = withArchivedChannels
+	opts.IncludeProfilePictures = includeProfilePictures
 	if err := a.BulkExport(rctx, fileWriter, filepath.Dir(outPath), nil /* nil job since it's spawned from CLI */, opts); err != nil {
 		CommandPrintErrorln(err.Error())
 		return err

server/cmd/mmctl/commands/export.go

@@ -101,6 +101,7 @@ func init() {
 
 	ExportCreateCmd.Flags().Bool("no-attachments", false, "Exclude file attachments from the export file.")
 	ExportCreateCmd.Flags().Bool("include-archived-channels", false, "Include archived channels in the export file.")
+	ExportCreateCmd.Flags().Bool("include-profile-pictures", false, "Include profile pictures in the export file.")
 
 	ExportDownloadCmd.Flags().Bool("resume", false, "Set to true to resume an export download.")
 	_ = ExportDownloadCmd.Flags().MarkHidden("resume")
@@ -142,6 +143,11 @@ func exportCreateCmdF(c client.Client, command *cobra.Command, args []string) er
 		data["include_archived_channels"] = "true"
 	}
 
+	includeProfilePictures, _ := command.Flags().GetBool("include-profile-pictures")
+	if includeProfilePictures {
+		data["include_profile_pictures"] = "true"
+	}
+
 	job, _, err := c.CreateJob(context.TODO(), &model.Job{
 		Type: model.JobTypeExportProcess,
 		Data: data,

server/cmd/mmctl/docs/mmctl_export_create.rst

@@ -22,6 +22,7 @@ Options
 
   -h, --help                        help for create
       --include-archived-channels   Include archived channels in the export file.
+      --include-profile-pictures    Include profile pictures in the export file.
       --no-attachments              Exclude file attachments from the export file.
 
 Options inherited from parent commands

server/i18n/en.json

@@ -4370,6 +4370,10 @@
     "id": "api.user.get_authorization_code.endpoint.app_error",
     "translation": "Error retrieving endpoint from Discovery Document."
   },
+  {
+    "id": "api.user.get_profile_image_path.app_error",
+    "translation": "Error while checking if a user has a custom profile image."
+  },
   {
     "id": "api.user.get_uploads_for_user.forbidden.app_error",
     "translation": "Failed to get uploads."

server/public/model/bulk_export.go

@@ -9,6 +9,7 @@ const ExportDataDir = "data"
 
 type BulkExportOpts struct {
 	IncludeAttachments      bool
+	IncludeProfilePictures  bool
 	IncludeArchivedChannels bool
 	CreateArchive           bool
 }