IntenseWebs/mattermost
3
0
Fork 0
mirror of https://github.com/mattermost/mattermost.git synced 2025-02-25 18:55:24 -06:00
Code Issues Packages Projects Releases Wiki Activity

PLT-6558: Basic data retention job scheduler/worker implementation. (#7449)

Browse Source 
* PLT-7639: Batch delete methods for data retention.

* PLT-6558: Basic data retention job worker/scheduler implementation.
This commit is contained in:
George Goldberg
2017-09-15 17:53:48 +01:00
committed by Christopher Speller
parent fd878bd50c
commit 7243aa6751
7 changed files with 82 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
config/default.json
View File

@@ -316,7 +316,11 @@
"PostsAggregatorJobStartTime": "03:00"
},
"DataRetentionSettings": {
"Enable": false
"EnableMessageDeletion": false,
"EnableFileDeletion": false,
"MessageRetentionDays": 365,
"FileRetentionDays": 365,
"DeletionJobStartTime": "02:00"
},
"JobSettings": {
"RunJobs": true,

12
i18n/en.json
View File

@@ -4355,6 +4355,18 @@
"id": "model.config.is_valid.elastic_search.aggregate_posts_after_days.app_error",
"translation": "Elasticsearch AggregatePostsAfterDays setting must be a number greater than or equal to 1"
},
{
"id": "model.config.is_valid.data_retention.message_retention_days_too_low.app_error",
"translation": "DataRetention MessageRetentionDays setting must be at least 1"
},
{
"id": "model.config.is_valid.data_retention.file_retention_days_too_low.app_error",
"translation": "DataRetention FileRetentionDays setting must be at least 1"
},
{
"id": "model.config.is_valid.data_retention.deletion_job_start_time.app_error",
"translation": "DataRetention DeletionJobStartTime setting must be set to a time in the format \"hh:mm\""
},
{
"id": "model.config.is_valid.elastic_search.connection_url.app_error",
"translation": "Elastic Search ConnectionUrl setting must be provided when Elastic Search indexing is enabled."

8
jobs/schedulers.go
View File

@@ -40,7 +40,7 @@ func (schedulers *Schedulers) Start() *Schedulers {
l4g.Info("Starting schedulers")
schedulers.startOnce.Do(func() {
if schedulers.DataRetention != nil && *utils.Cfg.DataRetentionSettings.Enable {
if schedulers.DataRetention != nil && (*utils.Cfg.DataRetentionSettings.EnableMessageDeletion || *utils.Cfg.DataRetentionSettings.EnableFileDeletion) {
go schedulers.DataRetention.Run()
}
@@ -56,9 +56,9 @@ func (schedulers *Schedulers) Start() *Schedulers {
func (schedulers *Schedulers) handleConfigChange(oldConfig *model.Config, newConfig *model.Config) {
if schedulers.DataRetention != nil {
if !*oldConfig.DataRetentionSettings.Enable && *newConfig.DataRetentionSettings.Enable {
if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion) {
go schedulers.DataRetention.Run()
} else if *oldConfig.DataRetentionSettings.Enable && !*newConfig.DataRetentionSettings.Enable {
} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion) {
schedulers.DataRetention.Stop()
}
}
@@ -75,7 +75,7 @@ func (schedulers *Schedulers) handleConfigChange(oldConfig *model.Config, newCon
func (schedulers *Schedulers) Stop() *Schedulers {
utils.RemoveConfigListener(schedulers.listenerId)
if schedulers.DataRetention != nil && *utils.Cfg.DataRetentionSettings.Enable {
if schedulers.DataRetention != nil && (*utils.Cfg.DataRetentionSettings.EnableMessageDeletion || *utils.Cfg.DataRetentionSettings.EnableFileDeletion) {
schedulers.DataRetention.Stop()
}

8
jobs/workers.go
View File

@@ -46,7 +46,7 @@ func (workers *Workers) Start() *Workers {
l4g.Info("Starting workers")
workers.startOnce.Do(func() {
if workers.DataRetention != nil && *utils.Cfg.DataRetentionSettings.Enable {
if workers.DataRetention != nil && (*utils.Cfg.DataRetentionSettings.EnableMessageDeletion || *utils.Cfg.DataRetentionSettings.EnableFileDeletion) {
go workers.DataRetention.Run()
}
@@ -68,9 +68,9 @@ func (workers *Workers) Start() *Workers {
func (workers *Workers) handleConfigChange(oldConfig *model.Config, newConfig *model.Config) {
if workers.DataRetention != nil {
if !*oldConfig.DataRetentionSettings.Enable && *newConfig.DataRetentionSettings.Enable {
if (!*oldConfig.DataRetentionSettings.EnableMessageDeletion && !*oldConfig.DataRetentionSettings.EnableFileDeletion) && (*newConfig.DataRetentionSettings.EnableMessageDeletion || *newConfig.DataRetentionSettings.EnableFileDeletion) {
go workers.DataRetention.Run()
} else if *oldConfig.DataRetentionSettings.Enable && !*newConfig.DataRetentionSettings.Enable {
} else if (*oldConfig.DataRetentionSettings.EnableMessageDeletion || *oldConfig.DataRetentionSettings.EnableFileDeletion) && (!*newConfig.DataRetentionSettings.EnableMessageDeletion && !*newConfig.DataRetentionSettings.EnableFileDeletion) {
workers.DataRetention.Stop()
}
}
@@ -97,7 +97,7 @@ func (workers *Workers) Stop() *Workers {
workers.watcher.Stop()
if workers.DataRetention != nil && *utils.Cfg.DataRetentionSettings.Enable {
if workers.DataRetention != nil && (*utils.Cfg.DataRetentionSettings.EnableMessageDeletion || *utils.Cfg.DataRetentionSettings.EnableFileDeletion) {
workers.DataRetention.Stop()
}

48
model/config.go
View File

@@ -138,6 +138,10 @@ const (
ELASTICSEARCH_SETTINGS_DEFAULT_POST_INDEX_SHARDS = 1
ELASTICSEARCH_SETTINGS_DEFAULT_AGGREGATE_POSTS_AFTER_DAYS = 365
ELASTICSEARCH_SETTINGS_DEFAULT_POSTS_AGGREGATOR_JOB_START_TIME = "03:00"
DATA_RETENTION_SETTINGS_DEFAULT_MESSAGE_RETENTION_DAYS = 365
DATA_RETENTION_SETTINGS_DEFAULT_FILE_RETENTION_DAYS = 365
DATA_RETENTION_SETTINGS_DEFAULT_DELETION_JOB_START_TIME = "02:00"
)
type ServiceSettings struct {
@@ -480,7 +484,11 @@ type ElasticsearchSettings struct {
}
type DataRetentionSettings struct {
Enable *bool
EnableMessageDeletion *bool
EnableFileDeletion *bool
MessageRetentionDays *int
FileRetentionDays *int
DeletionJobStartTime *string
}
type JobSettings struct {
@@ -1555,9 +1563,29 @@ func (o *Config) SetDefaults() {
*o.ElasticsearchSettings.PostsAggregatorJobStartTime = ELASTICSEARCH_SETTINGS_DEFAULT_POSTS_AGGREGATOR_JOB_START_TIME
}
if o.DataRetentionSettings.Enable == nil {
o.DataRetentionSettings.Enable = new(bool)
*o.DataRetentionSettings.Enable = false
if o.DataRetentionSettings.EnableMessageDeletion == nil {
o.DataRetentionSettings.EnableMessageDeletion = new(bool)
*o.DataRetentionSettings.EnableMessageDeletion = false
}
if o.DataRetentionSettings.EnableFileDeletion == nil {
o.DataRetentionSettings.EnableFileDeletion = new(bool)
*o.DataRetentionSettings.EnableMessageDeletion = false
}
if o.DataRetentionSettings.MessageRetentionDays == nil {
o.DataRetentionSettings.MessageRetentionDays = new(int)
*o.DataRetentionSettings.MessageRetentionDays = DATA_RETENTION_SETTINGS_DEFAULT_MESSAGE_RETENTION_DAYS
}
if o.DataRetentionSettings.FileRetentionDays == nil {
o.DataRetentionSettings.FileRetentionDays = new(int)
*o.DataRetentionSettings.FileRetentionDays = DATA_RETENTION_SETTINGS_DEFAULT_FILE_RETENTION_DAYS
}
if o.DataRetentionSettings.DeletionJobStartTime == nil {
o.DataRetentionSettings.DeletionJobStartTime = new(string)
*o.DataRetentionSettings.DeletionJobStartTime = DATA_RETENTION_SETTINGS_DEFAULT_DELETION_JOB_START_TIME
}
if o.JobSettings.RunJobs == nil {
@@ -1816,6 +1844,18 @@ func (o *Config) IsValid() *AppError {
return NewAppError("Config.IsValid", "model.config.is_valid.elastic_search.posts_aggregator_job_start_time.app_error", nil, err.Error(), http.StatusBadRequest)
}
if *o.DataRetentionSettings.MessageRetentionDays <= 0 {
return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.message_retention_days_too_low.app_error", nil, "", http.StatusBadRequest)
}
if *o.DataRetentionSettings.FileRetentionDays <= 0 {
return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.file_retention_days_too_low.app_error", nil, "", http.StatusBadRequest)
}
if _, err := time.Parse("03:04", *o.DataRetentionSettings.DeletionJobStartTime); err != nil {
return NewAppError("Config.IsValid", "model.config.is_valid.data_retention.deletion_job_start_time.app_error", nil, err.Error(), http.StatusBadRequest)
}
return nil
}

7
model/license.go
View File

@@ -54,6 +54,7 @@ type Features struct {
Announcement *bool `json:"announcement"`
ThemeManagement *bool `json:"theme_management"`
EmailNotificationContents *bool `json:"email_notification_contents"`
DataRetention *bool `json:"data_retention"`
// after we enabled more features for webrtc we'll need to control them with this
FutureFeatures *bool `json:"future_features"`
@@ -74,6 +75,7 @@ func (f *Features) ToMap() map[string]interface{} {
"password": *f.PasswordRequirements,
"elastic_search": *f.Elasticsearch,
"email_notification_contents": *f.EmailNotificationContents,
"data_retention": *f.DataRetention,
"future": *f.FutureFeatures,
}
}
@@ -163,6 +165,11 @@ func (f *Features) SetDefaults() {
f.EmailNotificationContents = new(bool)
*f.EmailNotificationContents = *f.FutureFeatures
}
if f.DataRetention == nil {
f.DataRetention = new(bool)
*f.DataRetention = *f.FutureFeatures
}
}
func (l *License) IsExpired() bool {

7
model/license_test.go
View File

@@ -26,8 +26,9 @@ func TestLicenseFeaturesToMap(t *testing.T) {
CheckTrue(t, m["saml"].(bool))
CheckTrue(t, m["password"].(bool))
CheckTrue(t, m["elastic_search"].(bool))
CheckTrue(t, m["future"].(bool))
CheckTrue(t, m["email_notification_contents"].(bool))
CheckTrue(t, m["data_retention"].(bool))
CheckTrue(t, m["future"].(bool))
}
func TestLicenseFeaturesSetDefaults(t *testing.T) {
@@ -48,6 +49,7 @@ func TestLicenseFeaturesSetDefaults(t *testing.T) {
CheckTrue(t, *f.PasswordRequirements)
CheckTrue(t, *f.Elasticsearch)
CheckTrue(t, *f.EmailNotificationContents)
CheckTrue(t, *f.DataRetention)
CheckTrue(t, *f.FutureFeatures)
f = Features{}
@@ -67,6 +69,7 @@ func TestLicenseFeaturesSetDefaults(t *testing.T) {
*f.SAML = true
*f.PasswordRequirements = true
*f.Elasticsearch = true
*f.DataRetention = true
*f.EmailNotificationContents = true
f.SetDefaults()
@@ -85,6 +88,7 @@ func TestLicenseFeaturesSetDefaults(t *testing.T) {
CheckTrue(t, *f.PasswordRequirements)
CheckTrue(t, *f.Elasticsearch)
CheckTrue(t, *f.EmailNotificationContents)
CheckTrue(t, *f.DataRetention)
CheckFalse(t, *f.FutureFeatures)
}
@@ -166,6 +170,7 @@ func TestLicenseToFromJson(t *testing.T) {
CheckBool(t, *f1.SAML, *f.SAML)
CheckBool(t, *f1.PasswordRequirements, *f.PasswordRequirements)
CheckBool(t, *f1.Elasticsearch, *f.Elasticsearch)
CheckBool(t, *f1.DataRetention, *f.DataRetention)
CheckBool(t, *f1.FutureFeatures, *f.FutureFeatures)
invalid := `{"asdf`