[MM-30831] granular data retention wireup (#17417)

* pre-checkout commit * add API endpoints for retention policies * allow deleting multiple teams/channels from a policy in a single request * pre-checkout commit * add auditing in API functions * add permission checks * update the store layers * update storetest * add check constraint on PostDuration column * pre-checkout commit * add query to delete posts under the scope of a granular retention policy * add suggestions from sbishel * allow clients to specify channels/teams when creating a new policy * remove foreign keys referencing Channels and Teams tables * add checks for whether teams and channels exist * pre-checkout commit * remove data referencing the Posts table * pre-checkout commit * write data store tests * sort results of buildGetPoliciesQuery * add missing test cases for teams * pre-checkout commit * add Client4 methods for data retention policy endpoints * add uint and uint64 to app/layer_generators * make granular policies override global policies * fix lint errors * pre-checkout commit * add license to top of files * add tests for data store layer * add missing test cases for store layer * run make i18n-extract * add query to delete ChannelMemberHistory * work in progress * add test for old reply to old post * fix lint error * use COALESCE on each Posts column * begin implementing orphaned rows worker * split PR * pre-checkout commit * use RetentionPolicyWithTeamAndChannelCounts * update app and api layers * run make i18n-extract * add RetentionPolicy to retrylayer_test.go * Revert "split PR" This reverts commit b316f03dd3. * fix errors caused by revert * add suggestions from sbishel * fix copy-paste error * fix lint errors * pre-checkout commit * add function to delete orphaned rows * use -1 for infinite retention * remove check constraint * copy i18n entries from master * re-run tests with newer enterprise branch * add team data to channel list * add search for channels and teams in a policy * add store tests for channel and team search * add suggestions from mkraft * run make einterfaces-mocks * fix lint errors * add suggestions from mkraft * move removeOrphanedRows method to wireup branch * Revert "move removeOrphanedRows method to wireup branch" This reverts commit 94605c9b4a. * use DeleteOrphanedRows where possible * run make i18n-extract * use COMPLIANCE permissions * run make migrations-bindadta * clean up teams before test * fix tests for TestRetentionPolicyStore * add API endpoints for mobile * fix lint error * fix some of the lint errors * move user/data_retention endpoints to data_retention.go * Revert "fix some of the lint errors" This reverts commit b5b2dc2756. * add exclude_policy_constrained parameter for /channels and /teams * fix lint errors * add policy_id field to GET endpoints for channels and teams * use PolicyWithTeamID in RetentionPolicy layer * fix lint errors * run make i18n-extract * update mock call in telemetry_test.go * return status:OK in JSON instead of 204 * pre-checkout commit * add policy_id field on channels/teams * fix lint errors * use sq.Eq instead of '?' * use new subsection permissions * update channels and teams endpoints to use new subsection permissions * add extra search opts for channels in a policy * fix lint errors * allow negative post duration in patch * remove DELETE FROM query in retention policy tests * use *int64 for PostDuration * re-run CI tests * use 3-step deletion strategy for each table * fix lint errors * run make store-layers * re-run CI tests * add test with channel, team and global policies * use common function for SQL queries * add pagination test * use struct for args to common SQL function * fix lint errors * run make i18n-extract * check if Channels.TeamId is "" or nil * use three OR clauses * write separate genericRetentionPoliciesDeletion function * add config setting for BatchSize * add telemetry for BatchSize * use feature flag * add old i18n messages back in * re-run CI tests * update call signature in storetest * MM-30831: Adds constant for retention default batch size. * MM-30831: Removes comment re: optimization. * MM-30831: Converts days to milliseconds. * MM-30831: Reverts change to test. * Revert "MM-30831: Reverts change to test." This reverts commit 6d14275a1c. * Revert "MM-30831: Converts days to milliseconds." This reverts commit a0cb6ec09d. * MM-30831: Fixes tests. * MM-30381: Fix for change to method sig. Co-authored-by: Max Erenberg <max.erenberg@mattermost.com> Co-authored-by: Mattermod <mattermod@users.noreply.github.com> Co-authored-by: Martin Kraft <martin@upspin.org>
2025-02-25 18:55:24 -06:00 · 2021-06-23 07:55:12 -04:00
parent 71ef1c5386
commit 58d5d51f7a
30 changed files with 1824 additions and 66 deletions
--- a/store/timerlayer/timerlayer.go
+++ b/store/timerlayer/timerlayer.go
@@ -2158,6 +2158,22 @@ func (s *TimerLayerChannelStore) UserBelongsToChannels(userID string, channelIds
 	return result, err
 }

+func (s *TimerLayerChannelMemberHistoryStore) DeleteOrphanedRows(limit int) (int64, error) {
+	start := timemodule.Now()
+
+	result, err := s.ChannelMemberHistoryStore.DeleteOrphanedRows(limit)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("ChannelMemberHistoryStore.DeleteOrphanedRows", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerChannelMemberHistoryStore) GetUsersInChannelDuring(startTime int64, endTime int64, channelID string) ([]*model.ChannelMemberHistoryResult, error) {
 	start := timemodule.Now()

@@ -2222,6 +2238,22 @@ func (s *TimerLayerChannelMemberHistoryStore) PermanentDeleteBatch(endTime int64
 	return result, err
 }

+func (s *TimerLayerChannelMemberHistoryStore) PermanentDeleteBatchForRetentionPolicies(now int64, globalPolicyEndTime int64, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	start := timemodule.Now()
+
+	result, resultVar1, err := s.ChannelMemberHistoryStore.PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit, cursor)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("ChannelMemberHistoryStore.PermanentDeleteBatchForRetentionPolicies", success, elapsed)
+	}
+	return result, resultVar1, err
+}
+
 func (s *TimerLayerClusterDiscoveryStore) Cleanup() error {
 	start := timemodule.Now()

@@ -4586,6 +4618,22 @@ func (s *TimerLayerPostStore) Delete(postID string, time int64, deleteByID strin
 	return err
 }

+func (s *TimerLayerPostStore) DeleteOrphanedRows(limit int) (int64, error) {
+	start := timemodule.Now()
+
+	result, err := s.PostStore.DeleteOrphanedRows(limit)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("PostStore.DeleteOrphanedRows", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerPostStore) Get(ctx context.Context, id string, skipFetchThreads bool, collapsedThreads bool, collapsedThreadsExtended bool, userID string) (*model.PostList, error) {
 	start := timemodule.Now()

@@ -5033,6 +5081,22 @@ func (s *TimerLayerPostStore) PermanentDeleteBatch(endTime int64, limit int64) (
 	return result, err
 }

+func (s *TimerLayerPostStore) PermanentDeleteBatchForRetentionPolicies(now int64, globalPolicyEndTime int64, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	start := timemodule.Now()
+
+	result, resultVar1, err := s.PostStore.PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit, cursor)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("PostStore.PermanentDeleteBatchForRetentionPolicies", success, elapsed)
+	}
+	return result, resultVar1, err
+}
+
 func (s *TimerLayerPostStore) PermanentDeleteByChannel(channelID string) error {
 	start := timemodule.Now()

@@ -5209,6 +5273,22 @@ func (s *TimerLayerPreferenceStore) DeleteCategoryAndName(category string, name
 	return err
 }

+func (s *TimerLayerPreferenceStore) DeleteOrphanedRows(limit int) (int64, error) {
+	start := timemodule.Now()
+
+	result, err := s.PreferenceStore.DeleteOrphanedRows(limit)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("PreferenceStore.DeleteOrphanedRows", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerPreferenceStore) Get(userID string, category string, name string) (*model.Preference, error) {
 	start := timemodule.Now()

@@ -5401,6 +5481,22 @@ func (s *TimerLayerReactionStore) DeleteAllWithEmojiName(emojiName string) error
 	return err
 }

+func (s *TimerLayerReactionStore) DeleteOrphanedRows(limit int) (int64, error) {
+	start := timemodule.Now()
+
+	result, err := s.ReactionStore.DeleteOrphanedRows(limit)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("ReactionStore.DeleteOrphanedRows", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerReactionStore) GetForPost(postID string, allowFromCache bool) ([]*model.Reaction, error) {
 	start := timemodule.Now()

@@ -5625,6 +5721,22 @@ func (s *TimerLayerRetentionPolicyStore) Delete(id string) error {
 	return err
 }

+func (s *TimerLayerRetentionPolicyStore) DeleteOrphanedRows(limit int) (int64, error) {
+	start := timemodule.Now()
+
+	result, err := s.RetentionPolicyStore.DeleteOrphanedRows(limit)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("RetentionPolicyStore.DeleteOrphanedRows", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerRetentionPolicyStore) Get(id string) (*model.RetentionPolicyWithTeamAndChannelCounts, error) {
 	start := timemodule.Now()

@@ -7942,6 +8054,22 @@ func (s *TimerLayerThreadStore) DeleteMembershipForUser(userId string, postID st
 	return err
 }

+func (s *TimerLayerThreadStore) DeleteOrphanedRows(limit int) (int64, error) {
+	start := timemodule.Now()
+
+	result, err := s.ThreadStore.DeleteOrphanedRows(limit)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("ThreadStore.DeleteOrphanedRows", success, elapsed)
+	}
+	return result, err
+}
+
 func (s *TimerLayerThreadStore) Get(id string) (*model.Thread, error) {
 	start := timemodule.Now()

@@ -8118,6 +8246,38 @@ func (s *TimerLayerThreadStore) MarkAsRead(userID string, threadID string, times
 	return err
 }

+func (s *TimerLayerThreadStore) PermanentDeleteBatchForRetentionPolicies(now int64, globalPolicyEndTime int64, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	start := timemodule.Now()
+
+	result, resultVar1, err := s.ThreadStore.PermanentDeleteBatchForRetentionPolicies(now, globalPolicyEndTime, limit, cursor)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("ThreadStore.PermanentDeleteBatchForRetentionPolicies", success, elapsed)
+	}
+	return result, resultVar1, err
+}
+
+func (s *TimerLayerThreadStore) PermanentDeleteBatchThreadMembershipsForRetentionPolicies(now int64, globalPolicyEndTime int64, limit int64, cursor model.RetentionPolicyCursor) (int64, model.RetentionPolicyCursor, error) {
+	start := timemodule.Now()
+
+	result, resultVar1, err := s.ThreadStore.PermanentDeleteBatchThreadMembershipsForRetentionPolicies(now, globalPolicyEndTime, limit, cursor)
+
+	elapsed := float64(timemodule.Since(start)) / float64(timemodule.Second)
+	if s.Root.Metrics != nil {
+		success := "false"
+		if err == nil {
+			success = "true"
+		}
+		s.Root.Metrics.ObserveStoreMethodDuration("ThreadStore.PermanentDeleteBatchThreadMembershipsForRetentionPolicies", success, elapsed)
+	}
+	return result, resultVar1, err
+}
+
 func (s *TimerLayerThreadStore) Save(thread *model.Thread) (*model.Thread, error) {
 	start := timemodule.Now()