Implement auto tag on comments and videos

* Comments and videos can be automatically tagged using core rules or
   watched word lists
 * These tags can be used to automatically filter videos and comments
 * Introduce a new video comment policy where comments must be approved
   first
 * Comments may have to be approved if the user auto block them using
   core rules or watched word lists
 * Implement FEP-5624 to federate reply control policies

server/core/lib/user-import-export/exporters/auto-tag-policies.ts

@@ -0,0 +1,18 @@
+import { AutoTagPoliciesJSON } from '@peertube/peertube-models'
+import { AutomaticTagger } from '@server/lib/automatic-tags/automatic-tagger.js'
+import { AbstractUserExporter } from './abstract-user-exporter.js'
+
+export class AutoTagPoliciesExporter extends AbstractUserExporter <AutoTagPoliciesJSON> {
+
+  async export () {
+    const data = await AutomaticTagger.getAutomaticTagPolicies(this.user.Account)
+
+    return {
+      json: {
+        reviewComments: data.review.map(name => ({ name }))
+      } as AutoTagPoliciesJSON,
+
+      staticFiles: []
+    }
+  }
+}

server/core/lib/user-import-export/exporters/comments-exporter.ts

@@ -34,9 +34,9 @@ export class CommentsExporter extends AbstractUserExporter <CommentsExportJSON>
 
   private formatCommentsAP (comments: MCommentExport[]) {
     return Bluebird.mapSeries(comments, async ({ url }) => {
-      const comment = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoUrlAndAccount(url)
+      const comment = await VideoCommentModel.loadByUrlAndPopulateReplyAndVideoImmutableAndAccount(url)
 
-      const threadParentComments = await VideoCommentModel.listThreadParentComments(comment, undefined)
+      const threadParentComments = await VideoCommentModel.listThreadParentComments({ comment })
       let commentObject = comment.toActivityPubObject(threadParentComments) as VideoCommentObject
 
       const isPublic = true // Comments are always public

server/core/lib/user-import-export/exporters/index.ts

@@ -1,4 +1,6 @@
+export * from './abstract-user-exporter.js'
 export * from './account-exporter.js'
+export * from './auto-tag-policies.js'
 export * from './blocklist-exporter.js'
 export * from './channels-exporter.js'
 export * from './comments-exporter.js'
@@ -6,8 +8,8 @@ export * from './dislikes-exporter.js'
 export * from './followers-exporter.js'
 export * from './following-exporter.js'
 export * from './likes-exporter.js'
-export * from './abstract-user-exporter.js'
 export * from './user-settings-exporter.js'
 export * from './user-video-history-exporter.js'
 export * from './video-playlists-exporter.js'
 export * from './videos-exporter.js'
+export * from './watched-words-lists-exporter.js'

server/core/lib/user-import-export/exporters/videos-exporter.ts

@@ -1,5 +1,5 @@
 import { pick } from '@peertube/peertube-core-utils'
-import { ActivityCreate, FileStorage, VideoExportJSON, VideoObject, VideoPrivacy } from '@peertube/peertube-models'
+import { ActivityCreate, FileStorage, VideoCommentPolicy, VideoExportJSON, VideoObject, VideoPrivacy } from '@peertube/peertube-models'
 import { logger } from '@server/helpers/logger.js'
 import { USER_EXPORT_MAX_ITEMS } from '@server/initializers/constants.js'
 import { audiencify, getAudience } from '@server/lib/activitypub/audience.js'
@@ -151,7 +151,10 @@ export class VideosExporter extends AbstractUserExporter <VideoExportJSON> {
 
       nsfw: video.nsfw,
 
-      commentsEnabled: video.commentsEnabled,
+      commentsPolicy: video.commentsPolicy,
+      // TODO: remove, deprecated in 6.2
+      commentsEnabled: video.commentsPolicy !== VideoCommentPolicy.DISABLED,
+
       downloadEnabled: video.downloadEnabled,
 
       waitTranscoding: video.waitTranscoding,

server/core/lib/user-import-export/exporters/watched-words-lists-exporter.ts

@@ -0,0 +1,23 @@
+import { WatchedWordsListsJSON } from '@peertube/peertube-models'
+import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'
+import { AbstractUserExporter } from './abstract-user-exporter.js'
+
+export class WatchedWordsListsExporter extends AbstractUserExporter <WatchedWordsListsJSON> {
+
+  async export () {
+    const data = await WatchedWordsListModel.listForExport({ accountId: this.user.Account.id })
+
+    return {
+      json: {
+        watchedWordLists: data.map(list => ({
+          listName: list.listName,
+          words: list.words,
+          createdAt: list.createdAt.toISOString(),
+          updatedAt: list.updatedAt.toISOString()
+        }))
+      } as WatchedWordsListsJSON,
+
+      staticFiles: []
+    }
+  }
+}

server/core/lib/user-import-export/importers/review-comments-tag-policies-importer.ts

@@ -0,0 +1,31 @@
+import { AutoTagPoliciesJSON, AutomaticTagPolicy } from '@peertube/peertube-models'
+import { isWatchedWordListNameValid } from '@server/helpers/custom-validators/watched-words.js'
+import { setAccountAutomaticTagsPolicy } from '@server/lib/automatic-tags/automatic-tags.js'
+import { AbstractUserImporter } from './abstract-user-importer.js'
+
+type SanitizedObject = AutoTagPoliciesJSON['reviewComments']
+
+// eslint-disable-next-line max-len
+export class ReviewCommentsTagPoliciesImporter
+  extends AbstractUserImporter <AutoTagPoliciesJSON, AutoTagPoliciesJSON['reviewComments'] & { archiveFiles?: never }, SanitizedObject> {
+
+  protected getImportObjects (json: AutoTagPoliciesJSON) {
+    if (!json.reviewComments) return []
+
+    return [ json.reviewComments ]
+  }
+
+  protected sanitize (data: AutoTagPoliciesJSON['reviewComments']) {
+    return data.filter(d => isWatchedWordListNameValid(d.name))
+  }
+
+  protected async importObject (data: SanitizedObject) {
+    await setAccountAutomaticTagsPolicy({
+      account: this.user.Account,
+      policy: AutomaticTagPolicy.REVIEW_COMMENT,
+      tags: data.map(v => v.name)
+    })
+
+    return { duplicate: false }
+  }
+}

server/core/lib/user-import-export/importers/user-video-history-importer.ts

@@ -1,14 +1,14 @@
-import { UserVideoHistoryExportJSON } from '@peertube/peertube-models'
-import { AbstractRatesImporter } from './abstract-rates-importer.js'
-import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
 import { pick } from '@peertube/peertube-core-utils'
+import { UserVideoHistoryExportJSON } from '@peertube/peertube-models'
+import { isUrlValid } from '@server/helpers/custom-validators/activitypub/misc.js'
 import { loadOrCreateVideoIfAllowedForUser } from '@server/lib/model-loaders/video.js'
 import { UserVideoHistoryModel } from '@server/models/user/user-video-history.js'
+import { AbstractUserImporter } from './abstract-user-importer.js'
 
-type SanitizedObject = Pick<UserVideoHistoryExportJSON['watchedVideos'][0], 'videoUrl' | 'lastTimecode'>
+type SanitizedObject = Pick<UserVideoHistoryExportJSON['watchedVideos'][0], 'videoUrl' | 'lastTimecode' | 'archiveFiles'>
 
 // eslint-disable-next-line max-len
-export class UserVideoHistoryImporter extends AbstractRatesImporter <UserVideoHistoryExportJSON, UserVideoHistoryExportJSON['watchedVideos'][0]> {
+export class UserVideoHistoryImporter extends AbstractUserImporter <UserVideoHistoryExportJSON, UserVideoHistoryExportJSON['watchedVideos'][0], SanitizedObject> {
 
   protected getImportObjects (json: UserVideoHistoryExportJSON) {
     return json.watchedVideos

server/core/lib/user-import-export/importers/videos-importer.ts

@@ -1,6 +1,13 @@
 import { pick } from '@peertube/peertube-core-utils'
 import { ffprobePromise, getVideoStreamDuration } from '@peertube/peertube-ffmpeg'
-import { LiveVideoLatencyMode, ThumbnailType, VideoExportJSON, VideoPrivacy, VideoState } from '@peertube/peertube-models'
+import {
+  LiveVideoLatencyMode,
+  ThumbnailType,
+  VideoCommentPolicy,
+  VideoExportJSON,
+  VideoPrivacy,
+  VideoState
+} from '@peertube/peertube-models'
 import { buildUUID, getFileSize } from '@peertube/peertube-node-utils'
 import { isArray, isBooleanValid, isUUIDValid } from '@server/helpers/custom-validators/misc.js'
 import { isVideoCaptionLanguageValid } from '@server/helpers/custom-validators/video-captions.js'
@@ -10,6 +17,7 @@ import { isLiveLatencyModeValid } from '@server/helpers/custom-validators/video-
 import {
   isPasswordValid,
   isVideoCategoryValid,
+  isVideoCommentsPolicyValid,
   isVideoDescriptionValid,
   isVideoDurationValid,
   isVideoLanguageValid,
@@ -42,7 +50,7 @@ const lTags = loggerTagsFactory('user-import')
 
 type ImportObject = VideoExportJSON['videos'][0]
 type SanitizedObject = Pick<ImportObject, 'name' | 'duration' | 'channel' | 'privacy' | 'archiveFiles' | 'captions' | 'category' |
-'licence' | 'language' | 'description' | 'support' | 'nsfw' | 'isLive' | 'commentsEnabled' | 'downloadEnabled' | 'waitTranscoding' |
+'licence' | 'language' | 'description' | 'support' | 'nsfw' | 'isLive' | 'commentsPolicy' | 'downloadEnabled' | 'waitTranscoding' |
 'originallyPublishedAt' | 'tags' | 'live' | 'passwords' | 'source' | 'chapters'>
 
 export class VideosImporter extends AbstractUserImporter <VideoExportJSON, ImportObject, SanitizedObject> {
@@ -59,17 +67,27 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
     if (o.isLive !== true && !o.archiveFiles?.videoFile) return undefined
 
     if (!isVideoCategoryValid(o.category)) o.category = null
-    if (!isVideoLicenceValid(o.licence)) o.licence = CONFIG.DEFAULTS.PUBLISH.LICENCE
+    if (!o.licence || !isVideoLicenceValid(o.licence)) o.licence = CONFIG.DEFAULTS.PUBLISH.LICENCE
     if (!isVideoLanguageValid(o.language)) o.language = null
     if (!isVideoDescriptionValid(o.description)) o.description = null
     if (!isVideoSupportValid(o.support)) o.support = null
 
     if (!isBooleanValid(o.nsfw)) o.nsfw = false
     if (!isBooleanValid(o.isLive)) o.isLive = false
-    if (!isBooleanValid(o.commentsEnabled)) o.commentsEnabled = CONFIG.DEFAULTS.PUBLISH.COMMENTS_ENABLED
     if (!isBooleanValid(o.downloadEnabled)) o.downloadEnabled = CONFIG.DEFAULTS.PUBLISH.DOWNLOAD_ENABLED
     if (!isBooleanValid(o.waitTranscoding)) o.waitTranscoding = true
 
+    if (!o.commentsPolicy || !isVideoCommentsPolicyValid(o.commentsPolicy)) {
+      // Fallback to deprecated property
+      if (isBooleanValid(o.commentsEnabled)) {
+        o.commentsPolicy = o.commentsEnabled === true
+          ? VideoCommentPolicy.ENABLED
+          : VideoCommentPolicy.DISABLED
+      } else {
+        o.commentsPolicy = CONFIG.DEFAULTS.PUBLISH.COMMENTS_POLICY
+      }
+    }
+
     if (!isVideoSourceFilenameValid(o.source?.inputFilename)) o.source = undefined
 
     if (!isVideoOriginallyPublishedAtValid(o.originallyPublishedAt)) o.originallyPublishedAt = null
@@ -89,7 +107,7 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
       if (!isBooleanValid(o.live.saveReplay)) o.live.saveReplay = false
       if (o.live.saveReplay && !isVideoReplayPrivacyValid(o.live.replaySettings.privacy)) return undefined
 
-      if (!isLiveLatencyModeValid(o.live.latencyMode)) o.live.latencyMode = LiveVideoLatencyMode.DEFAULT
+      if (!o.live.latencyMode || !isLiveLatencyModeValid(o.live.latencyMode)) o.live.latencyMode = LiveVideoLatencyMode.DEFAULT
 
       if (!o.live.streamKey) o.live.streamKey = buildUUID()
       else if (!isUUIDValid(o.live.streamKey)) return undefined
@@ -114,7 +132,7 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
       'support',
       'nsfw',
       'isLive',
-      'commentsEnabled',
+      'commentsPolicy',
       'downloadEnabled',
       'waitTranscoding',
       'originallyPublishedAt',
@@ -204,7 +222,7 @@ export class VideosImporter extends AbstractUserImporter <VideoExportJSON, Impor
           'isLive',
           'nsfw',
           'tags',
-          'commentsEnabled',
+          'commentsPolicy',
           'downloadEnabled',
           'waitTranscoding',
           'originallyPublishedAt'

server/core/lib/user-import-export/importers/watched-words-lists-importer.ts

@@ -0,0 +1,35 @@
+import { pick } from '@peertube/peertube-core-utils'
+import { WatchedWordsListsJSON } from '@peertube/peertube-models'
+import { areWatchedWordsValid, isWatchedWordListNameValid } from '@server/helpers/custom-validators/watched-words.js'
+import { WatchedWordsListModel } from '@server/models/watched-words/watched-words-list.js'
+import { AbstractUserImporter } from './abstract-user-importer.js'
+
+type SanitizedObject = Pick<WatchedWordsListsJSON['watchedWordLists'][0], 'listName' | 'words'>
+
+// eslint-disable-next-line max-len
+export class WatchedWordsListsImporter extends AbstractUserImporter <WatchedWordsListsJSON, WatchedWordsListsJSON['watchedWordLists'][0], SanitizedObject> {
+
+  protected getImportObjects (json: WatchedWordsListsJSON) {
+    return json.watchedWordLists
+  }
+
+  protected sanitize (data: WatchedWordsListsJSON['watchedWordLists'][0]) {
+    if (!isWatchedWordListNameValid(data.listName)) return undefined
+    if (!areWatchedWordsValid(data.words)) return undefined
+
+    return pick(data, [ 'listName', 'words' ])
+  }
+
+  protected async importObject (data: SanitizedObject) {
+    const accountId = this.user.Account.id
+    const existing = await WatchedWordsListModel.loadByListName({ listName: data.listName, accountId })
+
+    if (existing) {
+      await existing.updateList({ listName: data.listName, words: data.words })
+    } else {
+      await WatchedWordsListModel.createList({ accountId, listName: data.listName, words: data.words })
+    }
+
+    return { duplicate: false }
+  }
+}

server/core/lib/user-import-export/user-exporter.ts

@@ -1,6 +1,23 @@
+import { FileStorage, UserExportState } from '@peertube/peertube-models'
+import { getFileSize } from '@peertube/peertube-node-utils'
+import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
+import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { UserModel } from '@server/models/user/user.js'
+import { MUserDefault, MUserExport } from '@server/types/models/index.js'
+import archiver, { Archiver } from 'archiver'
+import { createWriteStream } from 'fs'
+import { remove } from 'fs-extra/esm'
 import { join, parse } from 'path'
+import { PassThrough, Readable, Writable } from 'stream'
+import { activityPubCollection } from '../activitypub/collection.js'
+import { getContextFilter } from '../activitypub/context.js'
+import { getUserExportFileObjectStorageSize, removeUserExportObjectStorage, storeUserExportFile } from '../object-storage/user-export.js'
+import { getFSUserExportFilePath } from '../paths.js'
 import {
+  AbstractUserExporter,
   AccountExporter,
+  AutoTagPoliciesExporter,
   BlocklistExporter,
   ChannelsExporter,
   CommentsExporter,
@@ -8,27 +25,13 @@ import {
   ExportResult,
   FollowersExporter,
   FollowingExporter,
-  LikesExporter, AbstractUserExporter,
+  LikesExporter,
   UserSettingsExporter,
+  UserVideoHistoryExporter,
   VideoPlaylistsExporter,
   VideosExporter,
-  UserVideoHistoryExporter
+  WatchedWordsListsExporter
 } from './exporters/index.js'
-import { MUserDefault, MUserExport } from '@server/types/models/index.js'
-import archiver, { Archiver } from 'archiver'
-import { createWriteStream } from 'fs'
-import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
-import { PassThrough, Readable, Writable } from 'stream'
-import { activityPubContextify } from '@server/helpers/activity-pub-utils.js'
-import { getContextFilter } from '../activitypub/context.js'
-import { activityPubCollection } from '../activitypub/collection.js'
-import { FileStorage, UserExportState } from '@peertube/peertube-models'
-import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
-import { UserModel } from '@server/models/user/user.js'
-import { getFSUserExportFilePath } from '../paths.js'
-import { getUserExportFileObjectStorageSize, removeUserExportObjectStorage, storeUserExportFile } from '../object-storage/user-export.js'
-import { getFileSize } from '@peertube/peertube-node-utils'
-import { remove } from 'fs-extra/esm'
 
 const lTags = loggerTagsFactory('user-export')
 
@@ -245,6 +248,14 @@ export class UserExporter {
       {
         jsonFilename: 'video-history.json',
         exporter: new UserVideoHistoryExporter(options)
+      },
+      {
+        jsonFilename: 'watched-words-lists.json',
+        exporter: new WatchedWordsListsExporter(options)
+      },
+      {
+        jsonFilename: 'automatic-tag-policies.json',
+        exporter: new AutoTagPoliciesExporter(options)
       }
     ] as { jsonFilename: string, exporter: AbstractUserExporter<any> }[]
   }

server/core/lib/user-import-export/user-importer.ts

@@ -1,23 +1,25 @@
-import { MUserDefault, MUserImport } from '@server/types/models/index.js'
-import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
 import { UserImportResultSummary, UserImportState } from '@peertube/peertube-models'
-import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
-import { getFSUserImportFilePath } from '../paths.js'
-import { remove } from 'fs-extra/esm'
-import { unzip } from '@server/helpers/unzip.js'
 import { getFilenameWithoutExt } from '@peertube/peertube-node-utils'
-import { VideosImporter } from './importers/videos-importer.js'
+import { saveInTransactionWithRetries } from '@server/helpers/database-utils.js'
+import { logger, loggerTagsFactory } from '@server/helpers/logger.js'
+import { unzip } from '@server/helpers/unzip.js'
 import { UserModel } from '@server/models/user/user.js'
+import { MUserDefault, MUserImport } from '@server/types/models/index.js'
+import { remove } from 'fs-extra/esm'
 import { dirname, join } from 'path'
-import { AccountImporter } from './importers/account-importer.js'
-import { UserSettingsImporter } from './importers/user-settings-importer.js'
-import { ChannelsImporter } from './importers/channels-importer.js'
+import { getFSUserImportFilePath } from '../paths.js'
 import { BlocklistImporter } from './importers/account-blocklist-importer.js'
+import { AccountImporter } from './importers/account-importer.js'
+import { ChannelsImporter } from './importers/channels-importer.js'
+import { DislikesImporter } from './importers/dislikes-importer.js'
 import { FollowingImporter } from './importers/following-importer.js'
 import { LikesImporter } from './importers/likes-importer.js'
-import { DislikesImporter } from './importers/dislikes-importer.js'
-import { VideoPlaylistsImporter } from './importers/video-playlists-importer.js'
+import { ReviewCommentsTagPoliciesImporter } from './importers/review-comments-tag-policies-importer.js'
+import { UserSettingsImporter } from './importers/user-settings-importer.js'
 import { UserVideoHistoryImporter } from './importers/user-video-history-importer.js'
+import { VideoPlaylistsImporter } from './importers/video-playlists-importer.js'
+import { VideosImporter } from './importers/videos-importer.js'
+import { WatchedWordsListsImporter } from './importers/watched-words-lists-importer.js'
 
 const lTags = loggerTagsFactory('user-import')
 
@@ -36,7 +38,9 @@ export class UserImporter {
         videos: this.buildSummary(),
         account: this.buildSummary(),
         userSettings: this.buildSummary(),
-        userVideoHistory: this.buildSummary()
+        userVideoHistory: this.buildSummary(),
+        watchedWordsLists: this.buildSummary(),
+        commentAutoTagPolicies: this.buildSummary()
       }
     }
 
@@ -133,6 +137,14 @@ export class UserImporter {
       {
         name: 'userVideoHistory' as 'userVideoHistory',
         importer: new UserVideoHistoryImporter(this.buildImporterOptions(user, 'video-history.json'))
+      },
+      {
+        name: 'watchedWordsLists' as 'watchedWordsLists',
+        importer: new WatchedWordsListsImporter(this.buildImporterOptions(user, 'watched-words-lists.json'))
+      },
+      {
+        name: 'commentAutoTagPolicies' as 'commentAutoTagPolicies',
+        importer: new ReviewCommentsTagPoliciesImporter(this.buildImporterOptions(user, 'automatic-tag-policies.json'))
       }
     ]
   }