XYZ-35: Added Support for GlobalRelay Compliance Export Format

* Added username to ChannelMemberHistory struct in anticipation of supporting GlobalRelay in Compliance Export
* Removed translation from debug output - this makes it complicated to use utils functions from tests in the enterprise repo
* Added an advanced email function that allows for greater control over message details. Updated MessageExport config to support GlobalRelay. Added attachment support to InBucket unit tests
* Moving templates in from enterprise to solve test issues
* Added export format to diagnostics
* Changed email attachment code to use FileBackend so that S3 storage is properly supported

model/channel_member_history.go

@@ -6,7 +6,10 @@ package model
 type ChannelMemberHistory struct {
 	ChannelId string
 	UserId    string
-	UserEmail string `db:"Email"`
 	JoinTime  int64
 	LeaveTime *int64
+
+	// these two fields are never set in the database - when we SELECT, we join on Users to get them
+	UserEmail string `db:"Email"`
+	Username  string
 }

model/config.go

@@ -158,6 +158,9 @@ const (
 
 	PLUGIN_SETTINGS_DEFAULT_DIRECTORY        = "./plugins"
 	PLUGIN_SETTINGS_DEFAULT_CLIENT_DIRECTORY = "./client/plugins"
+
+	COMPLIANCE_EXPORT_TYPE_ACTIANCE    = "actiance"
+	COMPLIANCE_EXPORT_TYPE_GLOBALRELAY = "globalrelay"
 )
 
 type ServiceSettings struct {
@@ -1623,9 +1626,13 @@ func (s *PluginSettings) SetDefaults() {
 
 type MessageExportSettings struct {
 	EnableExport        *bool
+	ExportFormat        *string
 	DailyRunTime        *string
 	ExportFromTimestamp *int64
 	BatchSize           *int
+
+	// formatter-specific settings - these are only expected to be non-nil if ExportFormat is set to the associated format
+	GlobalRelayEmailAddress *string
 }
 
 func (s *MessageExportSettings) SetDefaults() {
@@ -1633,6 +1640,10 @@ func (s *MessageExportSettings) SetDefaults() {
 		s.EnableExport = NewBool(false)
 	}
 
+	if s.ExportFormat == nil {
+		s.ExportFormat = NewString(COMPLIANCE_EXPORT_TYPE_ACTIANCE)
+	}
+
 	if s.DailyRunTime == nil {
 		s.DailyRunTime = NewString("01:00")
 	}
@@ -2170,6 +2181,16 @@ func (mes *MessageExportSettings) isValid(fs FileSettings) *AppError {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.daily_runtime.app_error", nil, err.Error(), http.StatusBadRequest)
 		} else if mes.BatchSize == nil || *mes.BatchSize < 0 {
 			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.batch_size.app_error", nil, "", http.StatusBadRequest)
+		} else if mes.ExportFormat == nil || (*mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_ACTIANCE && *mes.ExportFormat != COMPLIANCE_EXPORT_TYPE_GLOBALRELAY) {
+			return NewAppError("Config.IsValid", "model.config.is_valid.message_export.export_type.app_error", nil, "", http.StatusBadRequest)
+		}
+
+		if *mes.ExportFormat == COMPLIANCE_EXPORT_TYPE_GLOBALRELAY {
+			// validating email addresses is hard - just make sure it contains an '@' sign
+			// see https://stackoverflow.com/questions/201323/using-a-regular-expression-to-validate-an-email-address
+			if mes.GlobalRelayEmailAddress == nil || !strings.Contains(*mes.GlobalRelayEmailAddress, "@") {
+				return NewAppError("Config.IsValid", "model.config.is_valid.message_export.global_relay_email_address.app_error", nil, "", http.StatusBadRequest)
+			}
 		}
 	}
 	return nil

model/config_test.go

@@ -136,7 +136,7 @@ func TestMessageExportSettingsIsValidBatchSizeInvalid(t *testing.T) {
 	require.Error(t, mes.isValid(*fs))
 }
 
-func TestMessageExportSettingsIsValid(t *testing.T) {
+func TestMessageExportSettingsIsValidExportFormatInvalid(t *testing.T) {
 	fs := &FileSettings{
 		DriverName: NewString("foo"), // bypass file location check
 	}
@@ -147,6 +147,55 @@ func TestMessageExportSettingsIsValid(t *testing.T) {
 		BatchSize:           NewInt(100),
 	}
 
+	// should fail fast because export format isn't set
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidGlobalRelayEmailAddressInvalid(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFormat:        NewString(COMPLIANCE_EXPORT_TYPE_GLOBALRELAY),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		BatchSize:           NewInt(100),
+	}
+
+	// should fail fast because global relay email address isn't set
+	require.Error(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidActiance(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:        NewBool(true),
+		ExportFormat:        NewString(COMPLIANCE_EXPORT_TYPE_ACTIANCE),
+		ExportFromTimestamp: NewInt64(0),
+		DailyRunTime:        NewString("15:04"),
+		BatchSize:           NewInt(100),
+	}
+
+	// should pass because everything is valid
+	require.Nil(t, mes.isValid(*fs))
+}
+
+func TestMessageExportSettingsIsValidGlobalRelay(t *testing.T) {
+	fs := &FileSettings{
+		DriverName: NewString("foo"), // bypass file location check
+	}
+	mes := &MessageExportSettings{
+		EnableExport:            NewBool(true),
+		ExportFormat:            NewString(COMPLIANCE_EXPORT_TYPE_GLOBALRELAY),
+		ExportFromTimestamp:     NewInt64(0),
+		DailyRunTime:            NewString("15:04"),
+		BatchSize:               NewInt(100),
+		GlobalRelayEmailAddress: NewString("test@mattermost.com"),
+	}
+
 	// should pass because everything is valid
 	require.Nil(t, mes.isValid(*fs))
 }
@@ -159,6 +208,7 @@ func TestMessageExportSetDefaults(t *testing.T) {
 	require.Equal(t, "01:00", *mes.DailyRunTime)
 	require.Equal(t, int64(0), *mes.ExportFromTimestamp)
 	require.Equal(t, 10000, *mes.BatchSize)
+	require.Equal(t, COMPLIANCE_EXPORT_TYPE_ACTIANCE, *mes.ExportFormat)
 }
 
 func TestMessageExportSetDefaultsExportEnabledExportFromTimestampNil(t *testing.T) {

model/message_export.go

@@ -9,6 +9,7 @@ type MessageExport struct {
 
 	UserId    *string
 	UserEmail *string
+	Username  *string
 
 	PostId       *string
 	PostCreateAt *int64