PLT-7503: Create Message Export Scheduled Task and CLI Command (#7612)

* Created message export scheduled task

* Added CLI command to immediately kick off an export job

* Added email addresses for users joining and leaving the channel to the export

* Added support for both MySQL and PostgreSQL

* Fixing gofmt error

* Added a new ChannelMemberHistory store and associated tests

* Updating the ChannelMemberHistory channel as users create/join/leave channels

* Added user email to the message export object so it can be included in the actiance export xml

* Don't fail to log a leave event if a corresponding join event wasn't logged

* Adding copyright notices

* Adding message export settings to daily diagnostics report

* Added System Console integration for message export

* Cleaned up TODOs

* Made batch size configurable

* Added export from timestamp to CLI command

* Made ChannelMemberHistory table updates best effort

* Added a context-based timeout option to the message export CLI

* Minor PR updates/improvements

* Removed unnecessary fields from MessageExport object to reduce query overhead

* Removed JSON functions from the message export query in an effort to optimize performance

* Changed the way that channel member history queries and purges work to better account for edge cases

* Fixing a test I missed with the last refactor

* Added file copy functionality to file backend, improved config validation, added default config values

* Fixed file copy tests

* More concise use of the testing libraries

* Fixed context leak error

* Changed default export path to correctly place an 'export' directory under the 'data' directory

* Can't delete records from a read replica

* Fixed copy file tests

* Start job workers when license is applied, if configured to do so

* Suggestions from the PR

* Moar unit tests

* Fixed test imports

cmd/platform/mattermost.go

@@ -36,7 +36,7 @@ func init() {
 
 	resetCmd.Flags().Bool("confirm", false, "Confirm you really want to delete everything and a DB backup has been performed.")
 
-	rootCmd.AddCommand(serverCmd, versionCmd, userCmd, teamCmd, licenseCmd, importCmd, resetCmd, channelCmd, rolesCmd, testCmd, ldapCmd, configCmd, jobserverCmd, commandCmd)
+	rootCmd.AddCommand(serverCmd, versionCmd, userCmd, teamCmd, licenseCmd, importCmd, resetCmd, channelCmd, rolesCmd, testCmd, ldapCmd, configCmd, jobserverCmd, commandCmd, messageExportCmd)
 }
 
 var rootCmd = &cobra.Command{

cmd/platform/message_export.go

@@ -0,0 +1,79 @@
+// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"errors"
+
+	"context"
+
+	"time"
+
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/spf13/cobra"
+)
+
+var messageExportCmd = &cobra.Command{
+	Use:     "export",
+	Short:   "Export data from Mattermost",
+	Long:    "Export data from Mattermost in a format suitable for import into a third-party application",
+	Example: "export --format=actiance --exportFrom=12345",
+	RunE:    messageExportCmdF,
+}
+
+func init() {
+	messageExportCmd.Flags().String("format", "actiance", "The format to export data in")
+	messageExportCmd.Flags().Int64("exportFrom", -1, "The timestamp of the earliest post to export, expressed in seconds since the unix epoch.")
+	messageExportCmd.Flags().Int("timeoutSeconds", -1, "The maximum number of seconds to wait for the job to complete before timing out.")
+}
+
+func messageExportCmdF(cmd *cobra.Command, args []string) error {
+	a, err := initDBCommandContextCobra(cmd)
+	if err != nil {
+		return err
+	}
+
+	if !*a.Config().MessageExportSettings.EnableExport {
+		return errors.New("ERROR: The message export feature is not enabled")
+	}
+
+	// for now, format is hard-coded to actiance. In time, we'll have to support other formats and inject them into job data
+	if format, err := cmd.Flags().GetString("format"); err != nil {
+		return errors.New("format flag error")
+	} else if format != "actiance" {
+		return errors.New("unsupported export format")
+	}
+
+	startTime, err := cmd.Flags().GetInt64("exportFrom")
+	if err != nil {
+		return errors.New("exportFrom flag error")
+	} else if startTime < 0 {
+		return errors.New("exportFrom must be a positive integer")
+	}
+
+	timeoutSeconds, err := cmd.Flags().GetInt("timeoutSeconds")
+	if err != nil {
+		return errors.New("timeoutSeconds error")
+	} else if timeoutSeconds < 0 {
+		return errors.New("timeoutSeconds must be a positive integer")
+	}
+
+	if messageExportI := a.MessageExport; messageExportI != nil {
+		ctx := context.Background()
+		if timeoutSeconds > 0 {
+			var cancel context.CancelFunc
+			ctx, cancel = context.WithTimeout(ctx, time.Second*time.Duration(timeoutSeconds))
+			defer cancel()
+		}
+
+		job, err := messageExportI.StartSynchronizeJob(ctx, startTime)
+		if err != nil || job.Status == model.JOB_STATUS_ERROR || job.Status == model.JOB_STATUS_CANCELED {
+			CommandPrintErrorln("ERROR: Message export job failed. Please check the server logs")
+		} else {
+			CommandPrettyPrintln("SUCCESS: Message export job complete")
+		}
+	}
+
+	return nil
+}

cmd/platform/message_export_test.go

@@ -0,0 +1,65 @@
+// Copyright (c) 2016-present Mattermost, Inc. All Rights Reserved.
+// See License.txt for license information.
+
+package main
+
+import (
+	"testing"
+
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/mattermost/mattermost-server/model"
+	"github.com/mattermost/mattermost-server/utils"
+	"github.com/stretchr/testify/require"
+)
+
+// There are no tests that actually run the Message Export job, because it can take a long time to complete depending
+// on the size of the database that the config is pointing to. As such, these tests just ensure that the CLI command
+// fails fast if invalid flags are supplied
+
+func TestMessageExportNotEnabled(t *testing.T) {
+	configPath := writeTempConfig(t, false)
+	defer os.RemoveAll(filepath.Dir(configPath))
+
+	// should fail fast because the feature isn't enabled
+	require.Error(t, runCommand(t, "--config", configPath, "export"))
+}
+
+func TestMessageExportInvalidFormat(t *testing.T) {
+	configPath := writeTempConfig(t, true)
+	defer os.RemoveAll(filepath.Dir(configPath))
+
+	// should fail fast because format isn't supported
+	require.Error(t, runCommand(t, "--config", configPath, "--format", "not_actiance", "export"))
+}
+
+func TestMessageExportNegativeExportFrom(t *testing.T) {
+	configPath := writeTempConfig(t, true)
+	defer os.RemoveAll(filepath.Dir(configPath))
+
+	// should fail fast because export from must be a valid timestamp
+	require.Error(t, runCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "-1", "export"))
+}
+
+func TestMessageExportNegativeTimeoutSeconds(t *testing.T) {
+	configPath := writeTempConfig(t, true)
+	defer os.RemoveAll(filepath.Dir(configPath))
+
+	// should fail fast because timeout seconds must be a positive int
+	require.Error(t, runCommand(t, "--config", configPath, "--format", "actiance", "--exportFrom", "0", "--timeoutSeconds", "-1", "export"))
+}
+
+func writeTempConfig(t *testing.T, isMessageExportEnabled bool) string {
+	dir, err := ioutil.TempDir("", "")
+	require.NoError(t, err)
+
+	utils.TranslationsPreInit()
+	config := utils.LoadGlobalConfig("config.json")
+	config.MessageExportSettings.EnableExport = model.NewBool(isMessageExportEnabled)
+	configPath := filepath.Join(dir, "foo.json")
+	require.NoError(t, ioutil.WriteFile(configPath, []byte(config.ToJson()), 0600))
+
+	return configPath
+}